cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Debug CRISP naming

Browse Source
This commit is contained in:
Yam ZhengLim
2022-12-26 16:50:22 +08:00
parent d28dbff76a
commit 827499a309
4 changed files with 15 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
__pycache__/general_function.cpython-39.pyc
BIN
View File
Binary file not shown.
crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc
BIN
View File
Binary file not shown.
crisp/extract/crisp_extract_function.py
+12 -10
View File
@@ -18,12 +18,11 @@ def list_website_operators_csv(website_id, client, path, filename):
try: try:
output_location = path + filename output_location = path + filename
list_website_operators_json = client.website.list_website_operators(website_id) list_website_operators_json = client.website.list_website_operators(website_id)
list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json) list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json, sep='-')
# Datetime add # Datetime add
list_website_operators_df['api_called_at'] = date.today() - timedelta(days = 1) list_website_operators_df['api_called_at'] = date.today() - timedelta(days = 1)
list_website_operators_df.columns = list_website_operators_df.columns.str.replace(".", "-")
list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename) print("[Download Complete]",filename)
except Exception as e: except Exception as e:
@@ -41,7 +40,7 @@ def list_people_profiles_csv(website_id, client, path, filename):
while loop_flag == True: while loop_flag == True:
list_people_profiles_json = client.website.list_people_profiles(website_id, page_number) list_people_profiles_json = client.website.list_people_profiles(website_id, page_number)
temp_df = general_function.json_normalize_to_df(list_people_profiles_json) temp_df = general_function.json_normalize_to_df(list_people_profiles_json, sep='-')
temp_df['api_page_number'] = page_number temp_df['api_page_number'] = page_number
list_people_profiles_df = pd.concat([temp_df,list_people_profiles_df], ignore_index=True) list_people_profiles_df = pd.concat([temp_df,list_people_profiles_df], ignore_index=True)
if not list_people_profiles_json: if not list_people_profiles_json:
@@ -49,14 +48,13 @@ def list_people_profiles_csv(website_id, client, path, filename):
else: else:
page_number += 1 page_number += 1
# Timestamp to datetime # Timestamp to datetime
list_people_profiles_df['created_at']=(pd.to_datetime(list_people_profiles_df['created_at'],unit='s')) list_people_profiles_df['created_at']=(pd.to_datetime(list_people_profiles_df['created_at'],unit='ms'))
list_people_profiles_df['updated_at']=(pd.to_datetime(list_people_profiles_df['updated_at'],unit='s')) list_people_profiles_df['updated_at']=(pd.to_datetime(list_people_profiles_df['updated_at'],unit='ms'))
list_people_profiles_df['active-last']=(pd.to_datetime(list_people_profiles_df['active-last'],unit='s')) list_people_profiles_df['active-last']=(pd.to_datetime(list_people_profiles_df['active-last'],unit='ms'))
# Datetime add # Datetime add
list_people_profiles_df['api_called_at'] = date.today() - timedelta(days = 1) list_people_profiles_df['api_called_at'] = date.today() - timedelta(days = 1)
list_people_profiles_df.columns = list_people_profiles_df.columns.str.replace(".", "-")
list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename) print("[Download Complete]",filename)
@@ -77,14 +75,18 @@ def get_people_data_csv(website_id, client, path, filename, list_people_profiles
for people_id in person_ids: for people_id in person_ids:
get_people_data_json = client.website.get_people_data(website_id, people_id) get_people_data_json = client.website.get_people_data(website_id, people_id)
temp_df = general_function.json_normalize_to_df(get_people_data_json) temp_df = general_function.json_normalize_to_df(get_people_data_json)
temp_df['people_id'] = people_id temp_df['people_id'] = people_id
get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True) get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True)
# Fix setup error by CX team, eg( exchange-marking, shipping-marking -> should be shipping_marking)
get_people_data_df.columns = get_people_data_df.columns.str.replace("-", "_")
get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "-")
# Datetime add # Datetime add
get_people_data_df['api_called_at'] = date.today() - timedelta(days = 1) get_people_data_df['api_called_at'] = date.today() - timedelta(days = 1)
get_people_data_df.columns = get_people_data_df.columns.str.replace("-", "_")
get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "-")
get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename) print("[Download Complete]",filename)
@@ -105,7 +107,7 @@ def get_people_subscription_status_csv(website_id, client, path, filename, list_
# Get email subscription status # Get email subscription status
for people_id in person_ids: for people_id in person_ids:
pget_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id) pget_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json) temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json, sep='-')
temp_df['people_id'] = people_id temp_df['people_id'] = people_id
get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True) get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True)
general_function.py
+3 -2
View File
@@ -28,9 +28,10 @@ def change_to_current_directory():
os.chdir(script_dir) os.chdir(script_dir)
def json_normalize_to_df(json): def json_normalize_to_df(json, sep='.'):
sep_char = sep
try: try:
df = pd.json_normalize(json) df = pd.json_normalize(json, sep=sep_char)
return df return df
except Exception as e: except Exception as e:
print(e) print(e)