cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add timestamp

Browse Source
This commit is contained in:
Yam ZhengLim
2022-12-23 16:39:33 +08:00
parent 8439a617f1
commit 951e5e970e
2 changed files with 12 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc
BIN
View File
Binary file not shown.
crisp/extract/crisp_extract_function.py
+12
View File
@@ -1,6 +1,7 @@
from crisp_api import Crisp from crisp_api import Crisp
import general_function import general_function
import pandas as pd import pandas as pd
from datetime import date, timedelta, datetime
def connect_crisp(config): def connect_crisp(config):
try: try:
@@ -19,6 +20,9 @@ def list_website_operators_csv(website_id, client, path, filename):
list_website_operators_json = client.website.list_website_operators(website_id) list_website_operators_json = client.website.list_website_operators(website_id)
list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json) list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json)
# Datetime add
list_website_operators_df['api_called_at'] = date.today() - timedelta(days = 1)
list_website_operators_df.columns = list_website_operators_df.columns.str.replace(".", "-") list_website_operators_df.columns = list_website_operators_df.columns.str.replace(".", "-")
list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename) print("[Download Complete]",filename)
@@ -45,6 +49,9 @@ def list_people_profiles_csv(website_id, client, path, filename):
else: else:
page_number += 1 page_number += 1
# Datetime add
list_people_profiles_df['api_called_at'] = date.today() - timedelta(days = 1)
list_people_profiles_df.columns = list_people_profiles_df.columns.str.replace(".", "-") list_people_profiles_df.columns = list_people_profiles_df.columns.str.replace(".", "-")
list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename) print("[Download Complete]",filename)
@@ -69,6 +76,8 @@ def get_people_data_csv(website_id, client, path, filename, list_people_profiles
temp_df['people_id'] = people_id temp_df['people_id'] = people_id
get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True) get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True)
# Datetime add
get_people_data_df['api_called_at'] = date.today() - timedelta(days = 1)
get_people_data_df.columns = get_people_data_df.columns.str.replace("-", "_") get_people_data_df.columns = get_people_data_df.columns.str.replace("-", "_")
get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "-") get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "-")
@@ -96,6 +105,9 @@ def get_people_subscription_status_csv(website_id, client, path, filename, list_
temp_df['people_id'] = people_id temp_df['people_id'] = people_id
get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True) get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True)
# Datetime add
get_people_subscription_status_df['api_called_at'] = date.today() - timedelta(days = 1)
get_people_subscription_status_df.columns = get_people_subscription_status_df.columns.str.replace(".", "-") get_people_subscription_status_df.columns = get_people_subscription_status_df.columns.str.replace(".", "-")
get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
print("[Download Complete]",filename) print("[Download Complete]",filename)