cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Debug CRISP

Browse Source
This commit is contained in:
Yam ZhengLim
2022-12-08 00:22:59 +08:00
parent e1de2e3668
commit 5b3406e816
3 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc
BIN
View File
Binary file not shown.
crisp/extract/crisp_extract_function.py
+3 -1
View File
@@ -30,6 +30,7 @@ def get_peoples_csv(website_id, client, path, filename):
output_location = path + filename output_location = path + filename
peoples_list_df = pd.DataFrame() peoples_list_df = pd.DataFrame()
peoples_data_df = pd.DataFrame() peoples_data_df = pd.DataFrame()
people_subscription_status_df = pd.DataFrame()
loop_flag = True loop_flag = True
page_number = 1 page_number = 1
@@ -42,6 +43,7 @@ def get_peoples_csv(website_id, client, path, filename):
else: else:
page_number += 1 page_number += 1
# Get marking by CustomerService Operator # Get marking by CustomerService Operator
for people_id in peoples_list_df.people_id: for people_id in peoples_list_df.people_id:
people_data_json = client.website.get_people_data(website_id, people_id) people_data_json = client.website.get_people_data(website_id, people_id)
@@ -56,7 +58,7 @@ def get_peoples_csv(website_id, client, path, filename):
people_subscription_status_json = client.website.get_people_subscription_status(website_id, people_id) people_subscription_status_json = client.website.get_people_subscription_status(website_id, people_id)
temp_df = general_function.json_normalize_to_df(people_subscription_status_json) temp_df = general_function.json_normalize_to_df(people_subscription_status_json)
temp_df['people_id'] = people_id temp_df['people_id'] = people_id
people_subscription_status_df = pd.concat([temp_df,people_subscription_status_json], ignore_index=True) people_subscription_status_df = pd.concat([temp_df,people_subscription_status_df], ignore_index=True)
peoples_list_df = peoples_list_df.merge(people_subscription_status_df, on='people_id', how='left') peoples_list_df = peoples_list_df.merge(people_subscription_status_df, on='people_id', how='left')
main.py
+1 -1
View File
@@ -143,7 +143,7 @@ def main():
#Make sure cron in the file directory #Make sure cron in the file directory
general_function.change_to_current_directory() general_function.change_to_current_directory()
# #Extract Load Exchange Data #Extract Load Exchange Data
CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/" CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH) crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH) crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)