cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add c2m into extractload script

Browse Source
This commit is contained in:
Yam ZhengLim
2024-01-27 18:43:38 +08:00
parent 5d19b5a567
commit 25efe05695
1 changed files with 38 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+38 -38
View File
@@ -159,31 +159,31 @@ def c2m_load_to_cloudstorage(C2M_FILES_TEMP_STORAGE_PATH):
def main():
print("RUNNING MAIN_PY")
# # Exchange
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
# #Extract Load Exchange Data
# EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
# print("STARTING EXCHANGE EXTRACT")
# exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# print("STARTING EXCHANGE LOAD")
# exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# print("DELETING EXCHANGE CSV")
# general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# Exchange
#Make sure cron in the file directory
general_function.change_to_current_directory()
#Extract Load Exchange Data
EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
print("STARTING EXCHANGE EXTRACT")
exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
print("STARTING EXCHANGE LOAD")
exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
print("DELETING EXCHANGE CSV")
general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# # Shipping
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
# #Extract Load SHIPPING Data
# SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/"
# print("STARTING SHIPPING EXTRACT")
# shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
# print("STARTING SHIPPING LOAD")
# shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# print("DELETING SHIPPING CSV")
# general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
# Shipping
#Make sure cron in the file directory
general_function.change_to_current_directory()
#Extract Load SHIPPING Data
SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/"
print("STARTING SHIPPING EXTRACT")
shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
print("STARTING SHIPPING LOAD")
shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
print("DELETING SHIPPING CSV")
general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
# C2M
#Make sure cron in the file directory
@@ -198,21 +198,21 @@ def main():
print("DELETING SHIPPING CSV")
general_function.delete_csv_or_png_in_path(C2M_FILES_TEMP_STORAGE_PATH)
# # CRISP
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
# #Extract Load Exchange Data
# CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
# print("STARTING CRISP EXTRACT")
# crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
# print("STARTING CRISP LOAD")
# crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# print("DELETING CRISP CSV")
# general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
# CRISP
#Make sure cron in the file directory
general_function.change_to_current_directory()
#Extract Load Exchange Data
CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
print("STARTING CRISP EXTRACT")
crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
print("STARTING CRISP LOAD")
crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
print("DELETING CRISP CSV")
general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
# print("END MAIN_PY")
# print("------------------------------------------------------------------------------------------------")
print("END MAIN_PY")
print("------------------------------------------------------------------------------------------------")
if True:
main()