cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-30 17:54:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

C2M Scrape add

Browse Source
This commit is contained in:
Yam ZhengLim
2024-01-27 14:46:57 +08:00
parent e53a9817a6
commit 4ff602f2c6
9 changed files with 280 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+75 -35
View File
@@ -8,7 +8,8 @@ import shipping.extract.shipping_extract_function as shipping_extract_func
import shipping.load.shipping_load_function as shipping_load_func
import crisp.extract.crisp_extract_function as crisp_extract_func
import crisp.load.crisp_load_function as crisp_load_func
import c2m.scrape.c2m_scrape_app as c2m_scrape_app
import c2m.load.c2m_load_function as c2m_load_func
def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
#Config file name
@@ -127,52 +128,91 @@ def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
file_path = os.path.join(dirpath,file_name)
crisp_load_func.upload_to_bucket(cirsp_storage_client, file_name, file_path)
def c2m_scrape(C2M_FILES_TEMP_STORAGE_PATH):
C2M_CONFIG_FILE = "C2M_CONFIG.json"
C2M_FILES_TEMP_STORAGE_PATH = C2M_FILES_TEMP_STORAGE_PATH
#Read config file
C2M_CONFIG = general_function.read_json_file("./config/",C2M_CONFIG_FILE)
#Run the c2m_scrape_app
c2m_scrape_app.run_app(C2M_FILES_TEMP_STORAGE_PATH, C2M_CONFIG)
def c2m_load_to_cloudstorage(C2M_FILES_TEMP_STORAGE_PATH):
#Config file name
CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
#Connect to cloudstorage
c2m_storage_client = c2m_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
#Get the filename and path in temp csv storage
temp_file_list = []
for (dirpath, dirnames, filenames) in walk(C2M_FILES_TEMP_STORAGE_PATH):
temp_file_list.extend(filenames)
#Loop all the file and upload to cloudstorage
for file_name in temp_file_list:
file_path = os.path.join(dirpath,file_name)
c2m_load_func.upload_to_bucket(c2m_storage_client, file_name, file_path)
def main():
print("RUNNING MAIN_PY")
# Exchange
#Make sure cron in the file directory
general_function.change_to_current_directory()
#Extract Load Exchange Data
EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
print("STARTING EXCHANGE EXTRACT")
exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
print("STARTING EXCHANGE LOAD")
exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
print("DELETING EXCHANGE CSV")
general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# # Exchange
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
# #Extract Load Exchange Data
# EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
# print("STARTING EXCHANGE EXTRACT")
# exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# print("STARTING EXCHANGE LOAD")
# exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# print("DELETING EXCHANGE CSV")
# general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# Shipping
# # Shipping
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
# #Extract Load SHIPPING Data
# SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/"
# print("STARTING SHIPPING EXTRACT")
# shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
# print("STARTING SHIPPING LOAD")
# shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# print("DELETING SHIPPING CSV")
# general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
# C2M
#Make sure cron in the file directory
general_function.change_to_current_directory()
#Extract Load SHIPPING Data
SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/"
print("STARTING SHIPPING EXTRACT")
shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
print("STARTING SHIPPING LOAD")
shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)
#Scrape C2M Data
C2M_FILES_TEMP_STORAGE_PATH = "./c2m/scrape/c2m_temp_scrape_files/"
print("STARTING C2M SCRAPE")
c2m_scrape(C2M_FILES_TEMP_STORAGE_PATH)
print("STARTING C2M LOAD")
c2m_load_to_cloudstorage(C2M_FILES_TEMP_STORAGE_PATH)
#Delete Temp CSV
print("DELETING SHIPPING CSV")
general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
general_function.delete_csv_or_png_in_path(C2M_FILES_TEMP_STORAGE_PATH)
# CRISP
#Make sure cron in the file directory
general_function.change_to_current_directory()
#Extract Load Exchange Data
CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
print("STARTING CRISP EXTRACT")
crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
print("STARTING CRISP LOAD")
crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
print("DELETING CRISP CSV")
general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
# # CRISP
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
# #Extract Load Exchange Data
# CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
# print("STARTING CRISP EXTRACT")
# crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
# print("STARTING CRISP LOAD")
# crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# print("DELETING CRISP CSV")
# general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
print("END MAIN_PY")
print("------------------------------------------------------------------------------------------------")
# print("END MAIN_PY")
# print("------------------------------------------------------------------------------------------------")
if True:
main()