diff --git a/__pycache__/general_function.cpython-39.pyc b/__pycache__/general_function.cpython-39.pyc index a0d8b91..d681405 100644 Binary files a/__pycache__/general_function.cpython-39.pyc and b/__pycache__/general_function.cpython-39.pyc differ diff --git a/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc b/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc index 3754cc1..cb6096c 100644 Binary files a/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc and b/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc differ diff --git a/crisp/extract/crisp_extract_function.py b/crisp/extract/crisp_extract_function.py index 2d075ce..f36b83f 100644 --- a/crisp/extract/crisp_extract_function.py +++ b/crisp/extract/crisp_extract_function.py @@ -14,23 +14,25 @@ def connect_crisp(config): print("[FUNCTION_ERROR]-connect_crisp") return False -def list_website_operators_csv(website_id, client, path, filename): +def list_website_operators_json(website_id, client, path, filename): try: output_location = path + filename list_website_operators_json = client.website.list_website_operators(website_id) list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json, sep='__') # Datetime add - list_website_operators_df['api_called_at'] = date.today() - timedelta(days = 1) + list_website_operators_df['api_called_at'] = datetime.now() - list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + # list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + list_website_operators_df.to_json(output_location, orient='records', force_ascii = False) print("[Download Complete]",filename) + except Exception as e: print(e) - print("[FUNCTION_ERROR]-list_website_operators_csv") + print("[FUNCTION_ERROR]-list_website_operators_json") return False -def list_people_profiles_csv(website_id, client, path, filename): +def list_people_profiles_json(website_id, client, path, filename): try: output_location = path + filename list_people_profiles_df = pd.DataFrame() @@ -53,23 +55,24 @@ def list_people_profiles_csv(website_id, client, path, filename): list_people_profiles_df['active__last']=(pd.to_datetime(list_people_profiles_df['active__last'],unit='ms')) # Datetime add - list_people_profiles_df['api_called_at'] = date.today() - timedelta(days = 1) + list_people_profiles_df['api_called_at'] = datetime.now() - list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + # list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + list_people_profiles_df.to_json(output_location, orient='records', force_ascii = False) print("[Download Complete]",filename) except Exception as e: print(e) - print("[FUNCTION_ERROR]-list_people_profiles_csv") + print("[FUNCTION_ERROR]-list_people_profiles_json") return False -def get_people_data_csv(website_id, client, path, filename, list_people_profiles_csv_path): +def get_people_data_json(website_id, client, path, filename, list_people_profiles_json_path): try: output_location = path + filename get_people_data_df = pd.DataFrame() - person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id'] + person_ids = pd.read_json(list_people_profiles_json_path)['people_id'] # Get marking by CustomerService Operator for people_id in person_ids: @@ -85,40 +88,104 @@ def get_people_data_csv(website_id, client, path, filename, list_people_profiles get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "__") # Datetime add - get_people_data_df['api_called_at'] = date.today() - timedelta(days = 1) + get_people_data_df['api_called_at'] = datetime.now() - get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + # get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + get_people_data_df.to_json(output_location, orient='records', force_ascii = False) print("[Download Complete]",filename) except Exception as e: print(e) - print("[FUNCTION_ERROR]-get_people_data_csv") + print("[FUNCTION_ERROR]-get_people_data_json") return False -def get_people_subscription_status_csv(website_id, client, path, filename, list_people_profiles_csv_path): +def get_people_subscription_status_json(website_id, client, path, filename, list_people_profiles_json_path): try: output_location = path + filename get_people_subscription_status_df = pd.DataFrame() - person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id'] + person_ids = pd.read_json(list_people_profiles_json_path)['people_id'] # Get email subscription status for people_id in person_ids: - pget_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id) - temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json, sep='__') + get_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id) + temp_df = general_function.json_normalize_to_df(get_people_subscription_status_df_json, sep='__') temp_df['people_id'] = people_id get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True) # Datetime add - get_people_subscription_status_df['api_called_at'] = date.today() - timedelta(days = 1) + get_people_subscription_status_df['api_called_at'] = datetime.now() get_people_subscription_status_df.columns = get_people_subscription_status_df.columns.str.replace(".", "-") - get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + # get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + get_people_subscription_status_df.to_json(output_location, orient='records', force_ascii = False) + print("[Download Complete]",filename) except Exception as e: print(e) - print("[FUNCTION_ERROR]-get_people_subscription_status_csv") - return False \ No newline at end of file + print("[FUNCTION_ERROR]-get_people_subscription_status_json") + return False + + + +def list_conversations_json(website_id, client, path, filename): + try: + loop_flag = True + page_number = 1 + output_location = path + filename + conversations_list_df = pd.DataFrame() + + while loop_flag: + conversations_list_df_json = client.website.list_conversations(website_id, page_number) + + temp_df = general_function.json_normalize_to_df(conversations_list_df_json, sep='__') + conversations_list_df = pd.concat([conversations_list_df, temp_df], ignore_index=True) + + if conversations_list_df_json == []: + loop_flag = False + else: + page_number += 1 + + conversations_list_df.columns = conversations_list_df.columns.str.replace("-", "_") + conversations_list_df['api_called_at'] = datetime.now() + + conversations_list_df.to_json(output_location, orient='records', force_ascii = False) + print("[Download Complete]",filename) + + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-list_conversations_json") + return False + + +def get_message_in_conversations_json(website_id, client, path, filename, list_conversations_json_path): + try: + output_location = path + filename + get_message_in_conversations_df = pd.DataFrame() + session_ids = pd.read_json(list_conversations_json_path)['session_id'] + + for session_id in session_ids: + print(session_id) #NEED DELETE + + get_message_in_conversations_json = client.website.get_messages_in_conversation(website_id, session_id, "") + temp_df = general_function.json_normalize_to_df(get_message_in_conversations_json, sep='__') + get_message_in_conversations_df = pd.concat([temp_df, get_message_in_conversations_df], ignore_index=True) + + while (temp_df.shape[0] == 40): #Maximum can get 40 conversation per request, Request for date before if the temp_df is full + query = { "timestamp_before" : str(temp_df.at[0,'timestamp']) } + get_message_in_conversations_json = client.website.get_messages_in_conversation(website_id, session_id, query) + temp_df = pd.json_normalize(get_message_in_conversations_json, sep='__') + get_message_in_conversations_df = pd.concat([temp_df,get_message_in_conversations_df], ignore_index=True) + + get_message_in_conversations_df['api_called_at'] = datetime.now() + get_message_in_conversations_df.to_json(output_location, orient='records', force_ascii = False) + print("[Download Complete]",filename) + + + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-get_message_in_conversations_json") + return False \ No newline at end of file diff --git a/crisp/load/__pycache__/crisp_load_function.cpython-39.pyc b/crisp/load/__pycache__/crisp_load_function.cpython-39.pyc index 9dc5c97..ce65279 100644 Binary files a/crisp/load/__pycache__/crisp_load_function.cpython-39.pyc and b/crisp/load/__pycache__/crisp_load_function.cpython-39.pyc differ diff --git a/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc b/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc index c06031b..f22ae4a 100644 Binary files a/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc and b/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc differ diff --git a/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc b/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc index 5aebbb1..359d676 100644 Binary files a/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc and b/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc differ diff --git a/main.py b/main.py index 9f2fff4..b977aaa 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ import os from os import walk +import pandas as pd import general_function import exchange.extract.exchange_extract_function as exchange_extract_func import exchange.load.exchange_load_function as exchange_load_func @@ -92,16 +93,22 @@ def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH): CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG) #Get operators csv - crisp_extract_func.list_website_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_website_operators.csv') + crisp_extract_func.list_website_operators_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_website_operators.json') #Get users(people) csv - crisp_extract_func.list_people_profiles_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_people_profiles.csv') + crisp_extract_func.list_people_profiles_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_people_profiles.json') - # Get peoples manually marking by operator - crisp_extract_func.get_people_data_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv') + #Get peoples manually marking by operator + crisp_extract_func.get_people_data_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.json') - # Get peoples subscription status - crisp_extract_func.get_people_subscription_status_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv') + #Get peoples subscription status + crisp_extract_func.get_people_subscription_status_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.json') + + #Get current converstation list + crisp_extract_func.list_conversations_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_conversations.json') + + #Get message in conversations + crisp_extract_func.get_message_in_conversations_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_message_in_conversations.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_conversations.json') def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH): #Config file name @@ -168,4 +175,4 @@ def main(): print("------------------------------------------------------------------------------------------------") if True: - main() + main() \ No newline at end of file diff --git a/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc b/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc index bb26810..b2d8853 100644 Binary files a/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc and b/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc differ diff --git a/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc b/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc index b895fda..01aa2be 100644 Binary files a/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc and b/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc differ