cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add CRISP & ExchangeLog Data

Browse Source
This commit is contained in:
Yam ZhengLim
2022-12-19 15:19:37 +08:00
parent 5b3406e816
commit cc7cea07a5
6 changed files with 159 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+28 -22
View File
@@ -79,11 +79,11 @@ def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
#Config file name
CRISP_API_CONFIG_FILE = "CRISP_CONFIG.json"
CRISP_API_CONFIG_FILE = 'CRISP_CONFIG.json'
CRISP_CSV_TEMP_STORAGE_PATH = CRISP_CSV_TEMP_STORAGE_PATH
#Read config file
CRISP_API_CONFIG = general_function.read_json_file("./config/",CRISP_API_CONFIG_FILE)
CRISP_API_CONFIG = general_function.read_json_file('./config/',CRISP_API_CONFIG_FILE)
#Get website_id
CRISP_WEBSITE_ID = CRISP_API_CONFIG['website_id']
@@ -92,10 +92,16 @@ def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG)
#Get operators csv
crisp_extract_func.get_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_operators.csv')
crisp_extract_func.get_list_website_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_list_website_operators.csv')
#Get users(people) csv
crisp_extract_func.get_peoples_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_peoples.csv')
crisp_extract_func.get_list_people_profiles_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_list_people_profiles.csv')
# Get peoples manually marking by operator
crisp_extract_func.get_people_data_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.csv', './crisp/extract/crisp_temp_extract_csv/crisp_get_list_people_profiles.csv')
# Get peoples subscription status
crisp_extract_func.get_people_subscription_status_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.csv', './crisp/extract/crisp_temp_extract_csv/crisp_get_list_people_profiles.csv')
def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
#Config file name
@@ -117,27 +123,27 @@ def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
def main():
#Make sure cron in the file directory
general_function.change_to_current_directory()
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
#Extract Load Exchange Data
EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# #Extract Load Exchange Data
# EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
# exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
#Make sure cron in the file directory
general_function.change_to_current_directory()
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
#Extract Load SHIPPING Data
SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/"
shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)
# #Extract Load SHIPPING Data
# SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/"
# shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
# shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
#Make sure cron in the file directory
@@ -148,8 +154,8 @@ def main():
crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
if True:
main()