diff --git a/crisp/extract/crisp_extract_function.py b/crisp/extract/crisp_extract_function.py index ea5402c..bfa57e8 100644 --- a/crisp/extract/crisp_extract_function.py +++ b/crisp/extract/crisp_extract_function.py @@ -13,20 +13,21 @@ def connect_crisp(config): print("[FUNCTION_ERROR]-connect_crisp") return False -def get_list_website_operators_csv(website_id, client, path, filename): +def list_website_operators_csv(website_id, client, path, filename): try: output_location = path + filename list_website_operators_json = client.website.list_website_operators(website_id) list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json) + list_website_operators_df.columns = list_website_operators_df.columns.str.replace(".", "-") list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) print("[Download Complete]",filename) except Exception as e: print(e) - print("[FUNCTION_ERROR]-get_list_website_operators_csv") + print("[FUNCTION_ERROR]-list_website_operators_csv") return False -def get_list_people_profiles_csv(website_id, client, path, filename): +def list_people_profiles_csv(website_id, client, path, filename): try: output_location = path + filename list_people_profiles_df = pd.DataFrame() @@ -50,7 +51,7 @@ def get_list_people_profiles_csv(website_id, client, path, filename): except Exception as e: print(e) - print("[FUNCTION_ERROR]-get_list_people_profiles_csv") + print("[FUNCTION_ERROR]-list_people_profiles_csv") return False @@ -68,6 +69,9 @@ def get_people_data_csv(website_id, client, path, filename, list_people_profiles temp_df['people_id'] = people_id get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True) + + get_people_data_df.columns = get_people_data_df.columns.str.replace("-", "_") + get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "-") get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) print("[Download Complete]",filename) @@ -92,6 +96,7 @@ def get_people_subscription_status_csv(website_id, client, path, filename, list_ temp_df['people_id'] = people_id get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True) + get_people_subscription_status_df.columns = get_people_subscription_status_df.columns.str.replace(".", "-") get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) print("[Download Complete]",filename) diff --git a/main.py b/main.py index 4f64cfa..1969e37 100644 --- a/main.py +++ b/main.py @@ -92,16 +92,16 @@ def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH): CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG) #Get operators csv - crisp_extract_func.get_list_website_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_list_website_operators.csv') + crisp_extract_func.list_website_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_website_operators.csv') #Get users(people) csv - crisp_extract_func.get_list_people_profiles_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_list_people_profiles.csv') + crisp_extract_func.list_people_profiles_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_people_profiles.csv') # Get peoples manually marking by operator - crisp_extract_func.get_people_data_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.csv', './crisp/extract/crisp_temp_extract_csv/crisp_get_list_people_profiles.csv') + crisp_extract_func.get_people_data_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv') # Get peoples subscription status - crisp_extract_func.get_people_subscription_status_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.csv', './crisp/extract/crisp_temp_extract_csv/crisp_get_list_people_profiles.csv') + crisp_extract_func.get_people_subscription_status_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv') def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH): #Config file name