cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

update crisp function

Browse Source
This commit is contained in:
Yam ZhengLim
2022-12-22 15:38:51 +08:00
parent a3501cba2f
commit 8439a617f1
2 changed files with 13 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crisp/extract/crisp_extract_function.py
+9 -4
View File
@@ -13,20 +13,21 @@ def connect_crisp(config):
print("[FUNCTION_ERROR]-connect_crisp")
return False
def get_list_website_operators_csv(website_id, client, path, filename):
def list_website_operators_csv(website_id, client, path, filename):
try:
output_location = path + filename
list_website_operators_json = client.website.list_website_operators(website_id)
list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json)
list_website_operators_df.columns = list_website_operators_df.columns.str.replace(".", "-")
list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-get_list_website_operators_csv")
print("[FUNCTION_ERROR]-list_website_operators_csv")
return False
def get_list_people_profiles_csv(website_id, client, path, filename):
def list_people_profiles_csv(website_id, client, path, filename):
try:
output_location = path + filename
list_people_profiles_df = pd.DataFrame()
@@ -50,7 +51,7 @@ def get_list_people_profiles_csv(website_id, client, path, filename):
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-get_list_people_profiles_csv")
print("[FUNCTION_ERROR]-list_people_profiles_csv")
return False
@@ -68,6 +69,9 @@ def get_people_data_csv(website_id, client, path, filename, list_people_profiles
temp_df['people_id'] = people_id
get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True)
get_people_data_df.columns = get_people_data_df.columns.str.replace("-", "_")
get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "-")
get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename)
@@ -92,6 +96,7 @@ def get_people_subscription_status_csv(website_id, client, path, filename, list_
temp_df['people_id'] = people_id
get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True)
get_people_subscription_status_df.columns = get_people_subscription_status_df.columns.str.replace(".", "-")
get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename)
main.py
+4 -4
View File
@@ -92,16 +92,16 @@ def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG)
#Get operators csv
crisp_extract_func.get_list_website_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_list_website_operators.csv')
crisp_extract_func.list_website_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_website_operators.csv')
#Get users(people) csv
crisp_extract_func.get_list_people_profiles_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_list_people_profiles.csv')
crisp_extract_func.list_people_profiles_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_people_profiles.csv')
# Get peoples manually marking by operator
crisp_extract_func.get_people_data_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.csv', './crisp/extract/crisp_temp_extract_csv/crisp_get_list_people_profiles.csv')
crisp_extract_func.get_people_data_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv')
# Get peoples subscription status
crisp_extract_func.get_people_subscription_status_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.csv', './crisp/extract/crisp_temp_extract_csv/crisp_get_list_people_profiles.csv')
crisp_extract_func.get_people_subscription_status_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv')
def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
#Config file name