cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update debug

Browse Source
This commit is contained in:
Yam ZhengLim
2023-02-22 01:48:26 +08:00
parent a133311917
commit 1c5b65e79b
2 changed files with 26 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/SHIPPING_EXTRACT_SQL.json
+1 -1
View File
@@ -33,6 +33,6 @@
"transaction_logs" : "SELECT * FROM transaction_logs;",
"transaction_details" : "SELECT * FROM `transaction_details`;",
"transport_arrangements" : "SELECT * FROM transport_arrangements",
"users" : "SELECT * FROM users;",
"users" : "SELECT id, name, email, type, status, remember_token, active_at, created_at, updated_at, deleted_at FROM users;",
"user_email_verifications" : "SELECT * FROM user_email_verifications;"
}
main.py
+25 -24
View File
@@ -123,19 +123,20 @@ def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
def main():
# Exchange
print("RUNNING MAIN_PY")
#Make sure cron in the file directory
general_function.change_to_current_directory()
#Extract Load Exchange Data
EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
print("STARTING EXCHANGE EXTRACT")
exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
print("STARTING EXCHANGE LOAD")
exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
print("DELETING EXCHANGE CSV")
general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# Exchange
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
# #Extract Load Exchange Data
# EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
# print("STARTING EXCHANGE EXTRACT")
# exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# print("STARTING EXCHANGE LOAD")
# exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# print("DELETING EXCHANGE CSV")
# general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# Shipping
#Make sure cron in the file directory
@@ -150,18 +151,18 @@ def main():
print("DELETING SHIPPING CSV")
general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
# CRISP
#Make sure cron in the file directory
general_function.change_to_current_directory()
#Extract Load Exchange Data
CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
print("STARTING CRISP EXTRACT")
crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
print("STARTING CRISP LOAD")
crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
print("DELETING CRISP CSV")
general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
# # CRISP
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
# #Extract Load Exchange Data
# CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
# print("STARTING CRISP EXTRACT")
# crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
# print("STARTING CRISP LOAD")
# crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# print("DELETING CRISP CSV")
# general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
print("END MAIN_PY")
print("------------------------------------------------------------------------------------------------")