mirror of
https://gitlab.com/cief-data/data-management.git
synced 2026-08-19 04:13:57 +00:00
Add CRISP & ExchangeLog Data
This commit is contained in:
Yam ZhengLim
@@ -40,6 +40,51 @@
|
||||
"wallets" : "SELECT * FROM wallets;"
|
||||
}
|
||||
|
||||
{
|
||||
"activity_log" : "SELECT ID, LOG_NAME, DESCRIPTION, SUBJECT_ID, SUBJECT_TYPE, CAUSER_ID, CAUSER_TYPE, '**MASKING_PROPERTIES**' AS PROPERTIES, CREATED_AT, UPDATED_AT FROM activity_log;",
|
||||
"addresses" : "SELECT ID, COMPANY_ID, COUNTRY_ID, STATE_ID, DISTRICT_ID, POSTCODE, '**MASKING_STREET_ONE**'AS STREET_ONE, '**MASKING_STREET_TWO**'AS STREET_TWO, BILLING, DELETED_AT, CREATED_AT, UPDATED_AT FROM addresses;",
|
||||
"announcements" : "SELECT * FROM announcements;",
|
||||
"announcement_segment" : "SELECT * FROM announcement_segment;",
|
||||
"banks" : "SELECT ID, COMPANY_ID, '**MASKING_REFERENCE**' AS REFERENCE, BANK_NAME, MD5(CONCAT(HOLDER_NAME,LENGTH(HOLDER_NAME)*2022*2)) AS HOLDER_NAME, MD5(CONCAT(ACCOUNT_NO,LENGTH(ACCOUNT_NO)*2022*2)) AS ACCOUNT_NO, BANK_BRANCH, MD5(CONCAT(SWIFT,LENGTH(SWIFT)*2022*2)) AS SWIFT, MD5(CONCAT(SNAP,LENGTH(SNAP)*2022*2)) AS SNAP, TYPE, `DEFAULT`, STATUS, COUNTRY_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM banks;",
|
||||
"bank_logs" : "SELECT ID, BANK_ID, COMPANY_ID, '**MASKING_REFERENCE**' AS REFERENCE, BANK_NAME, MD5(CONCAT(HOLDER_NAME,LENGTH(HOLDER_NAME)*2022*2)) AS HOLDER_NAME, MD5(CONCAT(ACCOUNT_NO,LENGTH(ACCOUNT_NO)*2022*2)) AS ACCOUNT_NO, BANK_BRANCH, MD5(CONCAT(SWIFT,LENGTH(SWIFT)*2022*2)) AS SWIFT, MD5(CONCAT(SNAP,LENGTH(SNAP)*2022*2)) AS SNAP, TYPE, `DEFAULT`, STATUS, COUNTRY_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM bank_logs;",
|
||||
"bookings" : "SELECT * FROM bookings;",
|
||||
"companies" : "SELECT ID, '**MASKING_NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, DEBTOR, TYPE, BUSINESS_TYPE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM companies;",
|
||||
"contacts" : "SELECT ID, COMPANY_ID, COUNTRY_ID, '**MASKING_REFERENCE**' AS REFERENCE, MD5(CONCAT(PHONE,LENGTH(PHONE)*2022*2)) AS PHONE, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, MD5(CONCAT(WECHAT_ID,LENGTH(WECHAT_ID)*2022*2)) AS WECHAT_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM contacts;",
|
||||
"countries" : "SELECT * FROM countries;",
|
||||
"currencies" : "SELECT * FROM currencies;",
|
||||
"currency_logs" : "SELECT * FROM currency_logs;",
|
||||
"currency_rates" : "SELECT * FROM currency_rates;",
|
||||
"currency_rate_logs" : "SELECT * FROM currency_rate_logs;",
|
||||
"districts" : "SELECT * FROM districts;",
|
||||
"documents" : "SELECT ID, OWNER_TYPE, OWNER_ID, DOCUMENT_TYPE, '**MASKING_REFERENCE**' AS REFERENCE, STATUS, ISSUED_DATE, EXPIRED_DATE, APPROVER, APPROVAL_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM documents;",
|
||||
"employees" : "SELECT * FROM employees;",
|
||||
"failed_jobs" : "SELECT ID, CONNECTION, QUEUE, '**MASKING_PAYLOAD' AS PAYLOAD, '**MASKING_EXCEPTION**' AS EXCEPTION, FAILED_AT FROM failed_jobs;",
|
||||
"files" : "SELECT id, document_id, '**MASKING_FILE**', file_type, deleted_at, created_at, updated_at FROM files;",
|
||||
"groups" : "SELECT ID, '**MASKING_REFERENCE**' AS REFERENCE, ISSUER, RECEIVER, AMOUNT, ORIGINAL_AMOUNT, CURRENCY_ID, ORIGINAL_CURRENCY_ID, CURRENCY_RATE, TAX, SERVICE_CHARGE, STATUS, CREATED_AT, UPDATED_AT FROM groups;",
|
||||
"group_transactions" : "SELECT * FROM group_transactions;",
|
||||
"jobs" : "SELECT ID, QUEUE, '**MASKING_PAYLOAD**' AS PAYLOAD, ATTEMPTS, RESERVED_AT, AVAILABLE_AT, CREATED_AT FROM jobs;",
|
||||
"migrations" : "SELECT * FROM migrations;",
|
||||
"model_has_roles" : "SELECT * FROM model_has_roles;",
|
||||
"notifications" : "SELECT * FROM notifications;",
|
||||
"password_resets" : "SELECT ID, USER_ID, '**MASKING_TOKEN**' AS TOKEN, IS_EXPIRED, IS_COMPLETE, CREATED_AT, UPDATED_AT FROM password_resets;",
|
||||
"permissions" : "SELECT * FROM permissions;",
|
||||
"roles" : "SELECT * FROM roles;",
|
||||
"role_has_permissions" : "SELECT * FROM role_has_permissions;",
|
||||
"segments" : "SELECT * FROM segments;",
|
||||
"segment_companies" : "SELECT * FROM segment_companies;",
|
||||
"segment_constants" : "SELECT ID, SEGMENT_ID, NAME, REFERENCE, '**MASKING_DETAIL**' AS DETAIL, DELETED_AT, CREATED_AT, UPDATED_AT FROM segment_constants;",
|
||||
"service_types" : "SELECT * FROM service_types;",
|
||||
"states" : "SELECT * FROM states;",
|
||||
"transactions" : "SELECT ID, OWNER_TYPE, OWNER_ID, TYPE, ISSUER, RECEIVER, RECIPIENT_BANK_ACCOUNT_ID, PAYMENT_METHOD, '**PAYMENT_REFERENCE**' AS PAYMENT_REFERENCE, MD5(CONCAT(BILL_NO,LENGTH(BILL_NO)*2022*2)) AS BILL_NO, AMOUNT, ORIGINAL_AMOUNT, CURRENCY_ID, ORIGINAL_CURRENCY_ID, CURRENCY_RATE, TAX, SERVICE_CHARGE, EXPIRES_ON, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM transactions;",
|
||||
"transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', '<br>'),'|', '|'),'\\\\','\\') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', '<br>'),'|', '|'),'\\\\','\\') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;",
|
||||
"users" : "SELECT id, '**MASKING_NAME**' AS name, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
|
||||
"user_email_verifications" : "SELECT ID, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, '**MASKING_TOKEN**' AS TOKEN, IS_COMPLETE, IS_ACTIVE, IS_SENT, CREATED_AT, UPDATED_AT FROM user_email_verifications;",
|
||||
"wallets" : "SELECT ID, OWNER_TYPE, OWNER_ID, MD5(CONCAT(CODE,LENGTH(CODE)*2022*2)) AS CODE, CURRENCY_ID, AMOUNT, DELETED_AT, CREATED_AT, UPDATED_AT FROM wallets;"
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
###### Region: US
|
||||
###### Directory: /home/cief_data/business_intelligence/data-management
|
||||
###### Cronjob run at: Everyday 12:30AM
|
||||
###### Snowflake task run at: Everyday 2:00AM
|
||||
###### Crontab config: crontab -e
|
||||
###### Crontab log: grep CRON /var/log/syslog
|
||||
###### Python log: grep 'mycmd' /var/log/syslog
|
||||
|
||||
@@ -1,43 +1,44 @@
|
||||
{
|
||||
"activity_log" : "SELECT ID, LOG_NAME, DESCRIPTION, SUBJECT_ID, SUBJECT_TYPE, CAUSER_ID, CAUSER_TYPE, '**MASKING_PROPERTIES**' AS PROPERTIES, CREATED_AT, UPDATED_AT FROM activity_log;",
|
||||
"addresses" : "SELECT ID, COMPANY_ID, COUNTRY_ID, STATE_ID, DISTRICT_ID, POSTCODE, '**MASKING_STREET_ONE**'AS STREET_ONE, '**MASKING_STREET_TWO**'AS STREET_TWO, BILLING, DELETED_AT, CREATED_AT, UPDATED_AT FROM addresses;",
|
||||
"activity_log" : "SELECT * FROM activity_log;",
|
||||
"addresses" : "SELECT * FROM addresses;",
|
||||
"announcements" : "SELECT * FROM announcements;",
|
||||
"announcement_segment" : "SELECT * FROM announcement_segment;",
|
||||
"banks" : "SELECT ID, COMPANY_ID, '**MASKING_REFERENCE**' AS REFERENCE, BANK_NAME, MD5(CONCAT(HOLDER_NAME,LENGTH(HOLDER_NAME)*2022*2)) AS HOLDER_NAME, MD5(CONCAT(ACCOUNT_NO,LENGTH(ACCOUNT_NO)*2022*2)) AS ACCOUNT_NO, BANK_BRANCH, MD5(CONCAT(SWIFT,LENGTH(SWIFT)*2022*2)) AS SWIFT, MD5(CONCAT(SNAP,LENGTH(SNAP)*2022*2)) AS SNAP, TYPE, `DEFAULT`, STATUS, COUNTRY_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM banks;",
|
||||
"bank_logs" : "SELECT ID, BANK_ID, COMPANY_ID, '**MASKING_REFERENCE**' AS REFERENCE, BANK_NAME, MD5(CONCAT(HOLDER_NAME,LENGTH(HOLDER_NAME)*2022*2)) AS HOLDER_NAME, MD5(CONCAT(ACCOUNT_NO,LENGTH(ACCOUNT_NO)*2022*2)) AS ACCOUNT_NO, BANK_BRANCH, MD5(CONCAT(SWIFT,LENGTH(SWIFT)*2022*2)) AS SWIFT, MD5(CONCAT(SNAP,LENGTH(SNAP)*2022*2)) AS SNAP, TYPE, `DEFAULT`, STATUS, COUNTRY_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM bank_logs;",
|
||||
"banks" : "SELECT * FROM banks;",
|
||||
"bank_logs" : "SELECT * FROM bank_logs;",
|
||||
"bookings" : "SELECT * FROM bookings;",
|
||||
"companies" : "SELECT ID, '**MASKING_NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, DEBTOR, TYPE, BUSINESS_TYPE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM companies;",
|
||||
"contacts" : "SELECT ID, COMPANY_ID, COUNTRY_ID, '**MASKING_REFERENCE**' AS REFERENCE, MD5(CONCAT(PHONE,LENGTH(PHONE)*2022*2)) AS PHONE, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, MD5(CONCAT(WECHAT_ID,LENGTH(WECHAT_ID)*2022*2)) AS WECHAT_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM contacts;",
|
||||
"booking_logs" : "SELECT * FROM booking_logs",
|
||||
"companies" : "SELECT * FROM companies;",
|
||||
"contacts" : "SELECT * FROM contacts;",
|
||||
"countries" : "SELECT * FROM countries;",
|
||||
"currencies" : "SELECT * FROM currencies;",
|
||||
"currency_logs" : "SELECT * FROM currency_logs;",
|
||||
"currency_rates" : "SELECT * FROM currency_rates;",
|
||||
"currency_rate_logs" : "SELECT * FROM currency_rate_logs;",
|
||||
"districts" : "SELECT * FROM districts;",
|
||||
"documents" : "SELECT ID, OWNER_TYPE, OWNER_ID, DOCUMENT_TYPE, '**MASKING_REFERENCE**' AS REFERENCE, STATUS, ISSUED_DATE, EXPIRED_DATE, APPROVER, APPROVAL_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM documents;",
|
||||
"documents" : "SELECT * FROM documents;",
|
||||
"employees" : "SELECT * FROM employees;",
|
||||
"failed_jobs" : "SELECT ID, CONNECTION, QUEUE, '**MASKING_PAYLOAD' AS PAYLOAD, '**MASKING_EXCEPTION**' AS EXCEPTION, FAILED_AT FROM failed_jobs;",
|
||||
"files" : "SELECT id, document_id, '**MASKING_FILE**', file_type, deleted_at, created_at, updated_at FROM files;",
|
||||
"groups" : "SELECT ID, '**MASKING_REFERENCE**' AS REFERENCE, ISSUER, RECEIVER, AMOUNT, ORIGINAL_AMOUNT, CURRENCY_ID, ORIGINAL_CURRENCY_ID, CURRENCY_RATE, TAX, SERVICE_CHARGE, STATUS, CREATED_AT, UPDATED_AT FROM groups;",
|
||||
"failed_jobs" : "SELECT * FROM failed_jobs;",
|
||||
"files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '<br>'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
|
||||
"groups" : "SELECT * FROM groups;",
|
||||
"group_transactions" : "SELECT * FROM group_transactions;",
|
||||
"jobs" : "SELECT ID, QUEUE, '**MASKING_PAYLOAD**' AS PAYLOAD, ATTEMPTS, RESERVED_AT, AVAILABLE_AT, CREATED_AT FROM jobs;",
|
||||
"jobs" : "SELECT * FROM jobs;",
|
||||
"migrations" : "SELECT * FROM migrations;",
|
||||
"model_has_roles" : "SELECT * FROM model_has_roles;",
|
||||
"notifications" : "SELECT * FROM notifications;",
|
||||
"password_resets" : "SELECT ID, USER_ID, '**MASKING_TOKEN**' AS TOKEN, IS_EXPIRED, IS_COMPLETE, CREATED_AT, UPDATED_AT FROM password_resets;",
|
||||
"password_resets" : "SELECT * FROM password_resets;",
|
||||
"permissions" : "SELECT * FROM permissions;",
|
||||
"roles" : "SELECT * FROM roles;",
|
||||
"role_has_permissions" : "SELECT * FROM role_has_permissions;",
|
||||
"segments" : "SELECT * FROM segments;",
|
||||
"segment_companies" : "SELECT * FROM segment_companies;",
|
||||
"segment_constants" : "SELECT ID, SEGMENT_ID, NAME, REFERENCE, '**MASKING_DETAIL**' AS DETAIL, DELETED_AT, CREATED_AT, UPDATED_AT FROM segment_constants;",
|
||||
"segment_constants" : "SELECT * FROM segment_constants;",
|
||||
"service_types" : "SELECT * FROM service_types;",
|
||||
"states" : "SELECT * FROM states;",
|
||||
"transactions" : "SELECT ID, OWNER_TYPE, OWNER_ID, TYPE, ISSUER, RECEIVER, RECIPIENT_BANK_ACCOUNT_ID, PAYMENT_METHOD, '**PAYMENT_REFERENCE**' AS PAYMENT_REFERENCE, MD5(CONCAT(BILL_NO,LENGTH(BILL_NO)*2022*2)) AS BILL_NO, AMOUNT, ORIGINAL_AMOUNT, CURRENCY_ID, ORIGINAL_CURRENCY_ID, CURRENCY_RATE, TAX, SERVICE_CHARGE, EXPIRES_ON, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM transactions;",
|
||||
"transactions" : "SELECT * FROM transactions;",
|
||||
"transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', '<br>'),'|', '|'),'\\\\','\\') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', '<br>'),'|', '|'),'\\\\','\\') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;",
|
||||
"users" : "SELECT id, '**MASKING_NAME**' AS name, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
|
||||
"user_email_verifications" : "SELECT ID, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, '**MASKING_TOKEN**' AS TOKEN, IS_COMPLETE, IS_ACTIVE, IS_SENT, CREATED_AT, UPDATED_AT FROM user_email_verifications;",
|
||||
"wallets" : "SELECT ID, OWNER_TYPE, OWNER_ID, MD5(CONCAT(CODE,LENGTH(CODE)*2022*2)) AS CODE, CURRENCY_ID, AMOUNT, DELETED_AT, CREATED_AT, UPDATED_AT FROM wallets;"
|
||||
"transaction_logs" : "SELECT * FROM transaction_logs",
|
||||
"users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
|
||||
"user_email_verifications" : "SELECT * FROM user_email_verifications;",
|
||||
"wallets" : "SELECT * FROM wallets;",
|
||||
"wallet_logs" : "SELECT * FROM wallet_logs"
|
||||
}
|
||||
|
||||
|
||||
|
||||
Binary file not shown.
@@ -13,59 +13,89 @@ def connect_crisp(config):
|
||||
print("[FUNCTION_ERROR]-connect_crisp")
|
||||
return False
|
||||
|
||||
def get_operators_csv(website_id, client, path, filename):
|
||||
def get_list_website_operators_csv(website_id, client, path, filename):
|
||||
try:
|
||||
output_location = path + filename
|
||||
operators_list_json = client.website.list_website_operators(website_id)
|
||||
operators_list_df = general_function.json_normalize_to_df(operators_list_json)
|
||||
operators_list_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
list_website_operators_json = client.website.list_website_operators(website_id)
|
||||
list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json)
|
||||
list_website_operators_df.columns = list_website_operators_df.columns.str.replace(".", "-")
|
||||
list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
print("[Download Complete]",filename)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("[FUNCTION_ERROR]-get_operators_csv")
|
||||
print("[FUNCTION_ERROR]-get_list_website_operators_csv")
|
||||
return False
|
||||
|
||||
def get_peoples_csv(website_id, client, path, filename):
|
||||
def get_list_people_profiles_csv(website_id, client, path, filename):
|
||||
try:
|
||||
output_location = path + filename
|
||||
peoples_list_df = pd.DataFrame()
|
||||
peoples_data_df = pd.DataFrame()
|
||||
people_subscription_status_df = pd.DataFrame()
|
||||
list_people_profiles_df = pd.DataFrame()
|
||||
|
||||
loop_flag = True
|
||||
page_number = 1
|
||||
|
||||
while loop_flag == True:
|
||||
peoples_list_json = client.website.list_people_profiles(website_id, page_number)
|
||||
temp_df = general_function.json_normalize_to_df(peoples_list_json)
|
||||
peoples_list_df = pd.concat([temp_df,peoples_list_df], ignore_index=True)
|
||||
if not peoples_list_json:
|
||||
list_people_profiles_json = client.website.list_people_profiles(website_id, page_number)
|
||||
temp_df = general_function.json_normalize_to_df(list_people_profiles_json)
|
||||
temp_df['api_page_number'] = page_number
|
||||
list_people_profiles_df = pd.concat([temp_df,list_people_profiles_df], ignore_index=True)
|
||||
if not list_people_profiles_json:
|
||||
loop_flag = False
|
||||
else:
|
||||
page_number += 1
|
||||
|
||||
|
||||
# Get marking by CustomerService Operator
|
||||
for people_id in peoples_list_df.people_id:
|
||||
people_data_json = client.website.get_people_data(website_id, people_id)
|
||||
temp_df = general_function.json_normalize_to_df(people_data_json)
|
||||
temp_df['people_id'] = people_id
|
||||
peoples_data_df = pd.concat([temp_df,peoples_data_df], ignore_index=True)
|
||||
|
||||
peoples_list_df = peoples_list_df.merge(peoples_data_df, on='people_id', how='left')
|
||||
|
||||
# Get email subscription status
|
||||
for people_id in peoples_list_df.people_id:
|
||||
people_subscription_status_json = client.website.get_people_subscription_status(website_id, people_id)
|
||||
temp_df = general_function.json_normalize_to_df(people_subscription_status_json)
|
||||
temp_df['people_id'] = people_id
|
||||
people_subscription_status_df = pd.concat([temp_df,people_subscription_status_df], ignore_index=True)
|
||||
|
||||
peoples_list_df = peoples_list_df.merge(people_subscription_status_df, on='people_id', how='left')
|
||||
|
||||
peoples_list_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
list_people_profiles_df.columns = list_people_profiles_df.columns.str.replace(".", "-")
|
||||
list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
print("[Download Complete]",filename)
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("[FUNCTION_ERROR]-get_peoples_csv")
|
||||
print("[FUNCTION_ERROR]-get_list_people_profiles_csv")
|
||||
return False
|
||||
|
||||
|
||||
def get_people_data_csv(website_id, client, path, filename, list_people_profiles_csv_path):
|
||||
try:
|
||||
output_location = path + filename
|
||||
get_people_data_df = pd.DataFrame()
|
||||
|
||||
person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
|
||||
|
||||
# Get marking by CustomerService Operator
|
||||
for people_id in person_ids:
|
||||
get_people_data_json = client.website.get_people_data(website_id, people_id)
|
||||
temp_df = general_function.json_normalize_to_df(get_people_data_json)
|
||||
temp_df['people_id'] = people_id
|
||||
get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True)
|
||||
|
||||
get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
print("[Download Complete]",filename)
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("[FUNCTION_ERROR]-get_people_data_csv")
|
||||
return False
|
||||
|
||||
|
||||
|
||||
def get_people_subscription_status_csv(website_id, client, path, filename, list_people_profiles_csv_path):
|
||||
try:
|
||||
output_location = path + filename
|
||||
get_people_subscription_status_df = pd.DataFrame()
|
||||
|
||||
person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
|
||||
|
||||
# Get email subscription status
|
||||
for people_id in person_ids:
|
||||
pget_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
|
||||
temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json)
|
||||
temp_df['people_id'] = people_id
|
||||
get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True)
|
||||
|
||||
get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
print("[Download Complete]",filename)
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("[FUNCTION_ERROR]-get_people_subscription_status_csv")
|
||||
return False
|
||||
@@ -79,11 +79,11 @@ def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
|
||||
|
||||
def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
|
||||
#Config file name
|
||||
CRISP_API_CONFIG_FILE = "CRISP_CONFIG.json"
|
||||
CRISP_API_CONFIG_FILE = 'CRISP_CONFIG.json'
|
||||
CRISP_CSV_TEMP_STORAGE_PATH = CRISP_CSV_TEMP_STORAGE_PATH
|
||||
|
||||
#Read config file
|
||||
CRISP_API_CONFIG = general_function.read_json_file("./config/",CRISP_API_CONFIG_FILE)
|
||||
CRISP_API_CONFIG = general_function.read_json_file('./config/',CRISP_API_CONFIG_FILE)
|
||||
|
||||
#Get website_id
|
||||
CRISP_WEBSITE_ID = CRISP_API_CONFIG['website_id']
|
||||
@@ -92,10 +92,16 @@ def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
|
||||
CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG)
|
||||
|
||||
#Get operators csv
|
||||
crisp_extract_func.get_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_operators.csv')
|
||||
crisp_extract_func.get_list_website_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_list_website_operators.csv')
|
||||
|
||||
#Get users(people) csv
|
||||
crisp_extract_func.get_peoples_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_peoples.csv')
|
||||
crisp_extract_func.get_list_people_profiles_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_list_people_profiles.csv')
|
||||
|
||||
# Get peoples manually marking by operator
|
||||
crisp_extract_func.get_people_data_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.csv', './crisp/extract/crisp_temp_extract_csv/crisp_get_list_people_profiles.csv')
|
||||
|
||||
# Get peoples subscription status
|
||||
crisp_extract_func.get_people_subscription_status_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.csv', './crisp/extract/crisp_temp_extract_csv/crisp_get_list_people_profiles.csv')
|
||||
|
||||
def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
|
||||
#Config file name
|
||||
@@ -117,27 +123,27 @@ def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
|
||||
|
||||
|
||||
def main():
|
||||
#Make sure cron in the file directory
|
||||
general_function.change_to_current_directory()
|
||||
# #Make sure cron in the file directory
|
||||
# general_function.change_to_current_directory()
|
||||
|
||||
#Extract Load Exchange Data
|
||||
EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
|
||||
exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
|
||||
exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
|
||||
# #Extract Load Exchange Data
|
||||
# EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
|
||||
# exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
|
||||
# exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
#Delete Temp CSV
|
||||
general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
|
||||
# #Delete Temp CSV
|
||||
# general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
#Make sure cron in the file directory
|
||||
general_function.change_to_current_directory()
|
||||
# #Make sure cron in the file directory
|
||||
# general_function.change_to_current_directory()
|
||||
|
||||
#Extract Load SHIPPING Data
|
||||
SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/"
|
||||
shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
|
||||
shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)
|
||||
# #Extract Load SHIPPING Data
|
||||
# SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/"
|
||||
# shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
|
||||
# shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
#Delete Temp CSV
|
||||
general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
|
||||
# #Delete Temp CSV
|
||||
# general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
|
||||
#Make sure cron in the file directory
|
||||
@@ -148,8 +154,8 @@ def main():
|
||||
crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
|
||||
crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
#Delete Temp CSV
|
||||
general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
|
||||
# #Delete Temp CSV
|
||||
# general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
if True:
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user