cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Crisp Extract Load

Browse Source
This commit is contained in:
Yam ZhengLim
2023-02-23 22:31:41 +08:00
parent 596a522f08
commit 164e72d1d3
9 changed files with 102 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
__pycache__/general_function.cpython-39.pyc
BIN
View File
Binary file not shown.
crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc
BIN
View File
Binary file not shown.
crisp/extract/crisp_extract_function.py
+88 -21
View File
@@ -14,23 +14,25 @@ def connect_crisp(config):
print("[FUNCTION_ERROR]-connect_crisp")
return False
def list_website_operators_csv(website_id, client, path, filename):
def list_website_operators_json(website_id, client, path, filename):
try:
output_location = path + filename
list_website_operators_json = client.website.list_website_operators(website_id)
list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json, sep='__')
# Datetime add
list_website_operators_df['api_called_at'] = date.today() - timedelta(days = 1)
list_website_operators_df['api_called_at'] = datetime.now()
list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
# list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
list_website_operators_df.to_json(output_location, orient='records', force_ascii = False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-list_website_operators_csv")
print("[FUNCTION_ERROR]-list_website_operators_json")
return False
def list_people_profiles_csv(website_id, client, path, filename):
def list_people_profiles_json(website_id, client, path, filename):
try:
output_location = path + filename
list_people_profiles_df = pd.DataFrame()
@@ -53,23 +55,24 @@ def list_people_profiles_csv(website_id, client, path, filename):
list_people_profiles_df['active__last']=(pd.to_datetime(list_people_profiles_df['active__last'],unit='ms'))
# Datetime add
list_people_profiles_df['api_called_at'] = date.today() - timedelta(days = 1)
list_people_profiles_df['api_called_at'] = datetime.now()
list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
# list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
list_people_profiles_df.to_json(output_location, orient='records', force_ascii = False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-list_people_profiles_csv")
print("[FUNCTION_ERROR]-list_people_profiles_json")
return False
def get_people_data_csv(website_id, client, path, filename, list_people_profiles_csv_path):
def get_people_data_json(website_id, client, path, filename, list_people_profiles_json_path):
try:
output_location = path + filename
get_people_data_df = pd.DataFrame()
person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
person_ids = pd.read_json(list_people_profiles_json_path)['people_id']
# Get marking by CustomerService Operator
for people_id in person_ids:
@@ -85,40 +88,104 @@ def get_people_data_csv(website_id, client, path, filename, list_people_profiles
get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "__")
# Datetime add
get_people_data_df['api_called_at'] = date.today() - timedelta(days = 1)
get_people_data_df['api_called_at'] = datetime.now()
get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
# get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
get_people_data_df.to_json(output_location, orient='records', force_ascii = False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-get_people_data_csv")
print("[FUNCTION_ERROR]-get_people_data_json")
return False
def get_people_subscription_status_csv(website_id, client, path, filename, list_people_profiles_csv_path):
def get_people_subscription_status_json(website_id, client, path, filename, list_people_profiles_json_path):
try:
output_location = path + filename
get_people_subscription_status_df = pd.DataFrame()
person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
person_ids = pd.read_json(list_people_profiles_json_path)['people_id']
# Get email subscription status
for people_id in person_ids:
pget_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json, sep='__')
get_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
temp_df = general_function.json_normalize_to_df(get_people_subscription_status_df_json, sep='__')
temp_df['people_id'] = people_id
get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True)
# Datetime add
get_people_subscription_status_df['api_called_at'] = date.today() - timedelta(days = 1)
get_people_subscription_status_df['api_called_at'] = datetime.now()
get_people_subscription_status_df.columns = get_people_subscription_status_df.columns.str.replace(".", "-")
get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
# get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
get_people_subscription_status_df.to_json(output_location, orient='records', force_ascii = False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-get_people_subscription_status_csv")
return False
print("[FUNCTION_ERROR]-get_people_subscription_status_json")
return False
def list_conversations_json(website_id, client, path, filename):
try:
loop_flag = True
page_number = 1
output_location = path + filename
conversations_list_df = pd.DataFrame()
while loop_flag:
conversations_list_df_json = client.website.list_conversations(website_id, page_number)
temp_df = general_function.json_normalize_to_df(conversations_list_df_json, sep='__')
conversations_list_df = pd.concat([conversations_list_df, temp_df], ignore_index=True)
if conversations_list_df_json == []:
loop_flag = False
else:
page_number += 1
conversations_list_df.columns = conversations_list_df.columns.str.replace("-", "_")
conversations_list_df['api_called_at'] = datetime.now()
conversations_list_df.to_json(output_location, orient='records', force_ascii = False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-list_conversations_json")
return False
def get_message_in_conversations_json(website_id, client, path, filename, list_conversations_json_path):
try:
output_location = path + filename
get_message_in_conversations_df = pd.DataFrame()
session_ids = pd.read_json(list_conversations_json_path)['session_id']
for session_id in session_ids:
print(session_id) #NEED DELETE
get_message_in_conversations_json = client.website.get_messages_in_conversation(website_id, session_id, "")
temp_df = general_function.json_normalize_to_df(get_message_in_conversations_json, sep='__')
get_message_in_conversations_df = pd.concat([temp_df, get_message_in_conversations_df], ignore_index=True)
while (temp_df.shape[0] == 40): #Maximum can get 40 conversation per request, Request for date before if the temp_df is full
query = { "timestamp_before" : str(temp_df.at[0,'timestamp']) }
get_message_in_conversations_json = client.website.get_messages_in_conversation(website_id, session_id, query)
temp_df = pd.json_normalize(get_message_in_conversations_json, sep='__')
get_message_in_conversations_df = pd.concat([temp_df,get_message_in_conversations_df], ignore_index=True)
get_message_in_conversations_df['api_called_at'] = datetime.now()
get_message_in_conversations_df.to_json(output_location, orient='records', force_ascii = False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-get_message_in_conversations_json")
return False
crisp/load/__pycache__/crisp_load_function.cpython-39.pyc
BIN
View File
Binary file not shown.
exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc
BIN
View File
Binary file not shown.
exchange/load/__pycache__/exchange_load_function.cpython-39.pyc
BIN
View File
Binary file not shown.
main.py
+14 -7
View File
@@ -1,5 +1,6 @@
import os
from os import walk
import pandas as pd
import general_function
import exchange.extract.exchange_extract_function as exchange_extract_func
import exchange.load.exchange_load_function as exchange_load_func
@@ -92,16 +93,22 @@ def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG)
#Get operators csv
crisp_extract_func.list_website_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_website_operators.csv')
crisp_extract_func.list_website_operators_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_website_operators.json')
#Get users(people) csv
crisp_extract_func.list_people_profiles_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_people_profiles.csv')
crisp_extract_func.list_people_profiles_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_people_profiles.json')
# Get peoples manually marking by operator
crisp_extract_func.get_people_data_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv')
#Get peoples manually marking by operator
crisp_extract_func.get_people_data_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.json')
# Get peoples subscription status
crisp_extract_func.get_people_subscription_status_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv')
#Get peoples subscription status
crisp_extract_func.get_people_subscription_status_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.json')
#Get current converstation list
crisp_extract_func.list_conversations_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_conversations.json')
#Get message in conversations
crisp_extract_func.get_message_in_conversations_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_message_in_conversations.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_conversations.json')
def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
#Config file name
@@ -168,4 +175,4 @@ def main():
print("------------------------------------------------------------------------------------------------")
if True:
main()
main()
shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc
BIN
View File
Binary file not shown.
shipping/load/__pycache__/shipping_load_function.cpython-39.pyc
BIN
View File
Binary file not shown.