cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add shipping ExtractLoad

Browse Source
This commit is contained in:
Yam ZhengLim
2022-11-13 21:06:35 +08:00
parent 3f25d3debc
commit e940b729e9
8 changed files with 90 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+13 -11
View File
@@ -1,8 +1,10 @@
import os
from os import walk
import general_function
import exchange.extract.exchange_extract_function as extract_func
import exchange.load.exchange_load_function as load_func
import exchange.extract.exchange_extract_function as exchange_extract_func
import exchange.load.exchange_load_function as exchange_load_func
import shipping.extract.shipping_extract_function as shipping_extract_func
import shipping.load.shipping_load_function as shipping_load_func
def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
#Config file name
@@ -15,18 +17,18 @@ def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
EX_SQL_DICT = general_function.read_json_file("./config/",EX_SQL_FILE)
#Connect to mysql
EX_MYSQLDB = extract_func.connect_mysql(EX_MYSQL_CONFIG)
EX_MYSQLDB = exchange_extract_func.connect_mysql(EX_MYSQL_CONFIG)
EX_MYSQLCURSOR = EX_MYSQLDB.cursor()
#Run EX_SQL_DICT to get all data from mysql in csv format
extract_func.mysql_dict_sql_to_csv(EX_SQL_DICT, EXCHANGE_CSV_TEMP_STORAGE_PATH, EX_MYSQLDB, EX_MYSQLCURSOR)
exchange_extract_func.mysql_dict_sql_to_csv(EX_SQL_DICT, EXCHANGE_CSV_TEMP_STORAGE_PATH, EX_MYSQLDB, EX_MYSQLCURSOR)
def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH):
#Config file name
CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
#Connect to cloudstorage
ex_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
ex_storage_client = exchange_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
#Get the filename and path in temp csv storage
temp_file_list = []
@@ -36,7 +38,7 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH):
#Loop all the file and upload to cloudstorage
for file_name in temp_file_list:
file_path = os.path.join(dirpath,file_name)
load_func.upload_to_bucket(ex_storage_client, file_name, file_path)
exchange_load_func.upload_to_bucket(ex_storage_client, file_name, file_path)
@@ -51,11 +53,11 @@ def shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH):
SHIPPING_SQL_DICT = general_function.read_json_file("./config/",SHIPPING_SQL_FILE)
#Connect to mysql
SHIPPING_MYSQLDB = extract_func.connect_mysql(SHIPPING_MYSQL_CONFIG)
SHIPPING_MYSQLDB = shipping_extract_func.connect_mysql(SHIPPING_MYSQL_CONFIG)
SHIPPING_MYSQLCURSOR = SHIPPING_MYSQLDB.cursor()
#Run EX_SQL_DICT to get all data from mysql in csv format
extract_func.mysql_dict_sql_to_csv(SHIPPING_SQL_DICT, SHIPPING_CSV_TEMP_STORAGE_PATH, SHIPPING_MYSQLDB, SHIPPING_MYSQLCURSOR)
shipping_extract_func.mysql_dict_sql_to_csv(SHIPPING_SQL_DICT, SHIPPING_CSV_TEMP_STORAGE_PATH, SHIPPING_MYSQLDB, SHIPPING_MYSQLCURSOR)
def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
@@ -63,7 +65,7 @@ def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
#Connect to cloudstorage
shipping_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
shipping_storage_client = shipping_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
#Get the filename and path in temp csv storage
temp_file_list = []
@@ -73,7 +75,7 @@ def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
#Loop all the file and upload to cloudstorage
for file_name in temp_file_list:
file_path = os.path.join(dirpath,file_name)
load_func.upload_to_bucket(shipping_storage_client, file_name, file_path)
shipping_load_func.upload_to_bucket(shipping_storage_client, file_name, file_path)
@@ -94,7 +96,7 @@ def main():
general_function.change_to_current_directory()
#Extract Load SHIPPING Data
SHIPPING_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/"
shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)