cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add CRISP & ExchangeLog Data

Browse Source
This commit is contained in:
Yam ZhengLim
2022-12-19 15:19:37 +08:00
parent 5b3406e816
commit cc7cea07a5
6 changed files with 159 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crisp/extract/crisp_extract_function.py
+64 -34
View File
@@ -13,59 +13,89 @@ def connect_crisp(config):
print("[FUNCTION_ERROR]-connect_crisp")
return False
def get_operators_csv(website_id, client, path, filename):
def get_list_website_operators_csv(website_id, client, path, filename):
try:
output_location = path + filename
operators_list_json = client.website.list_website_operators(website_id)
operators_list_df = general_function.json_normalize_to_df(operators_list_json)
operators_list_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
list_website_operators_json = client.website.list_website_operators(website_id)
list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json)
list_website_operators_df.columns = list_website_operators_df.columns.str.replace(".", "-")
list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-get_operators_csv")
print("[FUNCTION_ERROR]-get_list_website_operators_csv")
return False
def get_peoples_csv(website_id, client, path, filename):
def get_list_people_profiles_csv(website_id, client, path, filename):
try:
output_location = path + filename
peoples_list_df = pd.DataFrame()
peoples_data_df = pd.DataFrame()
people_subscription_status_df = pd.DataFrame()
list_people_profiles_df = pd.DataFrame()
loop_flag = True
page_number = 1
while loop_flag == True:
peoples_list_json = client.website.list_people_profiles(website_id, page_number)
temp_df = general_function.json_normalize_to_df(peoples_list_json)
peoples_list_df = pd.concat([temp_df,peoples_list_df], ignore_index=True)
if not peoples_list_json:
list_people_profiles_json = client.website.list_people_profiles(website_id, page_number)
temp_df = general_function.json_normalize_to_df(list_people_profiles_json)
temp_df['api_page_number'] = page_number
list_people_profiles_df = pd.concat([temp_df,list_people_profiles_df], ignore_index=True)
if not list_people_profiles_json:
loop_flag = False
else:
page_number += 1
# Get marking by CustomerService Operator
for people_id in peoples_list_df.people_id:
people_data_json = client.website.get_people_data(website_id, people_id)
temp_df = general_function.json_normalize_to_df(people_data_json)
temp_df['people_id'] = people_id
peoples_data_df = pd.concat([temp_df,peoples_data_df], ignore_index=True)
peoples_list_df = peoples_list_df.merge(peoples_data_df, on='people_id', how='left')
# Get email subscription status
for people_id in peoples_list_df.people_id:
people_subscription_status_json = client.website.get_people_subscription_status(website_id, people_id)
temp_df = general_function.json_normalize_to_df(people_subscription_status_json)
temp_df['people_id'] = people_id
people_subscription_status_df = pd.concat([temp_df,people_subscription_status_df], ignore_index=True)
peoples_list_df = peoples_list_df.merge(people_subscription_status_df, on='people_id', how='left')
peoples_list_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
list_people_profiles_df.columns = list_people_profiles_df.columns.str.replace(".", "-")
list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-get_peoples_csv")
print("[FUNCTION_ERROR]-get_list_people_profiles_csv")
return False
def get_people_data_csv(website_id, client, path, filename, list_people_profiles_csv_path):
try:
output_location = path + filename
get_people_data_df = pd.DataFrame()
person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
# Get marking by CustomerService Operator
for people_id in person_ids:
get_people_data_json = client.website.get_people_data(website_id, people_id)
temp_df = general_function.json_normalize_to_df(get_people_data_json)
temp_df['people_id'] = people_id
get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True)
get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-get_people_data_csv")
return False
def get_people_subscription_status_csv(website_id, client, path, filename, list_people_profiles_csv_path):
try:
output_location = path + filename
get_people_subscription_status_df = pd.DataFrame()
person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
# Get email subscription status
for people_id in person_ids:
pget_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json)
temp_df['people_id'] = people_id
get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True)
get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename)
except Exception as e:
print(e)
print("[FUNCTION_ERROR]-get_people_subscription_status_csv")
return False