diff --git a/__pycache__/general_function.cpython-39.pyc b/__pycache__/general_function.cpython-39.pyc index 8e8548e..a0d8b91 100644 Binary files a/__pycache__/general_function.cpython-39.pyc and b/__pycache__/general_function.cpython-39.pyc differ diff --git a/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc b/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc index 832b3c1..3754cc1 100644 Binary files a/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc and b/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc differ diff --git a/crisp/extract/crisp_extract_function.py b/crisp/extract/crisp_extract_function.py index 7be42ce..7c71900 100644 --- a/crisp/extract/crisp_extract_function.py +++ b/crisp/extract/crisp_extract_function.py @@ -18,12 +18,11 @@ def list_website_operators_csv(website_id, client, path, filename): try: output_location = path + filename list_website_operators_json = client.website.list_website_operators(website_id) - list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json) + list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json, sep='-') # Datetime add list_website_operators_df['api_called_at'] = date.today() - timedelta(days = 1) - list_website_operators_df.columns = list_website_operators_df.columns.str.replace(".", "-") list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) print("[Download Complete]",filename) except Exception as e: @@ -41,7 +40,7 @@ def list_people_profiles_csv(website_id, client, path, filename): while loop_flag == True: list_people_profiles_json = client.website.list_people_profiles(website_id, page_number) - temp_df = general_function.json_normalize_to_df(list_people_profiles_json) + temp_df = general_function.json_normalize_to_df(list_people_profiles_json, sep='-') temp_df['api_page_number'] = page_number list_people_profiles_df = pd.concat([temp_df,list_people_profiles_df], ignore_index=True) if not list_people_profiles_json: @@ -49,14 +48,13 @@ def list_people_profiles_csv(website_id, client, path, filename): else: page_number += 1 # Timestamp to datetime - list_people_profiles_df['created_at']=(pd.to_datetime(list_people_profiles_df['created_at'],unit='s')) - list_people_profiles_df['updated_at']=(pd.to_datetime(list_people_profiles_df['updated_at'],unit='s')) - list_people_profiles_df['active-last']=(pd.to_datetime(list_people_profiles_df['active-last'],unit='s')) + list_people_profiles_df['created_at']=(pd.to_datetime(list_people_profiles_df['created_at'],unit='ms')) + list_people_profiles_df['updated_at']=(pd.to_datetime(list_people_profiles_df['updated_at'],unit='ms')) + list_people_profiles_df['active-last']=(pd.to_datetime(list_people_profiles_df['active-last'],unit='ms')) # Datetime add list_people_profiles_df['api_called_at'] = date.today() - timedelta(days = 1) - list_people_profiles_df.columns = list_people_profiles_df.columns.str.replace(".", "-") list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) print("[Download Complete]",filename) @@ -77,14 +75,18 @@ def get_people_data_csv(website_id, client, path, filename, list_people_profiles for people_id in person_ids: get_people_data_json = client.website.get_people_data(website_id, people_id) temp_df = general_function.json_normalize_to_df(get_people_data_json) + + temp_df['people_id'] = people_id get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True) + # Fix setup error by CX team, eg( exchange-marking, shipping-marking -> should be shipping_marking) + get_people_data_df.columns = get_people_data_df.columns.str.replace("-", "_") + get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "-") + # Datetime add get_people_data_df['api_called_at'] = date.today() - timedelta(days = 1) - get_people_data_df.columns = get_people_data_df.columns.str.replace("-", "_") - get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "-") get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) print("[Download Complete]",filename) @@ -105,7 +107,7 @@ def get_people_subscription_status_csv(website_id, client, path, filename, list_ # Get email subscription status for people_id in person_ids: pget_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id) - temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json) + temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json, sep='-') temp_df['people_id'] = people_id get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True) diff --git a/general_function.py b/general_function.py index f8ccf12..925640f 100644 --- a/general_function.py +++ b/general_function.py @@ -28,9 +28,10 @@ def change_to_current_directory(): os.chdir(script_dir) -def json_normalize_to_df(json): +def json_normalize_to_df(json, sep='.'): + sep_char = sep try: - df = pd.json_normalize(json) + df = pd.json_normalize(json, sep=sep_char) return df except Exception as e: print(e)