cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Change X1 to Shipping

Browse Source
This commit is contained in:
Yam ZhengLim
2022-11-13 18:00:42 +08:00
parent 4d3b345935
commit 3f25d3debc
3 changed files with 27 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/X1_CONFIG.json → config/SHIPPING_CONFIG.json
View File
config/X1_EXTRACT_SQL.json → config/SHIPPING_EXTRACT_SQL.json
+1 -1
View File
@@ -24,7 +24,7 @@
"packages" : "SELECT * FROM packages",
"packing_lists" : "SELECT * FROM packing_lists",
"password_resets" : "SELECT * FROM password_resets;",
"remarks" : "SELECT * FROM remarks",
"remarks" : "SELECT id, owner_type, owner_id, commenter_id, REPLACE(REPLACE(REPLACE(content, '\n', ''),'|', ''),'\\\\','') AS 'content',, deleted_at, created_at, updated_at FROM remarks",
"schedules" : "SELECT * FROM schedules",
"segments" : "SELECT * FROM segments;",
"segment_constants" : "SELECT * FROM segment_constants;",
main.py
+26 -28
View File
@@ -1,6 +1,5 @@
import os
from os import walk
import time
import general_function
import exchange.extract.exchange_extract_function as extract_func
import exchange.load.exchange_load_function as load_func
@@ -41,67 +40,66 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH):
def x1_extract(X1_CSV_TEMP_STORAGE_PATH):
def shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH):
#Config file name
X1_MYSQL_CONFIG_FILE = "X1_CONFIG.json"
X1_SQL_FILE = "X1_EXTRACT_SQL.json"
X1_CSV_TEMP_STORAGE_PATH = X1_CSV_TEMP_STORAGE_PATH
SHIPPING_MYSQL_CONFIG_FILE = "SHIPPING_CONFIG.json"
SHIPPING_SQL_FILE = "SHIPPING_EXTRACT_SQL.json"
SHIPPING_CSV_TEMP_STORAGE_PATH = SHIPPING_CSV_TEMP_STORAGE_PATH
#Read config file
X1_MYSQL_CONFIG = general_function.read_json_file("./config/",X1_MYSQL_CONFIG_FILE)
X1_SQL_DICT = general_function.read_json_file("./config/",X1_SQL_FILE)
SHIPPING_MYSQL_CONFIG = general_function.read_json_file("./config/",SHIPPING_MYSQL_CONFIG_FILE)
SHIPPING_SQL_DICT = general_function.read_json_file("./config/",SHIPPING_SQL_FILE)
#Connect to mysql
X1_MYSQLDB = extract_func.connect_mysql(X1_MYSQL_CONFIG)
X1_MYSQLCURSOR = X1_MYSQLDB.cursor()
SHIPPING_MYSQLDB = extract_func.connect_mysql(SHIPPING_MYSQL_CONFIG)
SHIPPING_MYSQLCURSOR = SHIPPING_MYSQLDB.cursor()
#Run EX_SQL_DICT to get all data from mysql in csv format
extract_func.mysql_dict_sql_to_csv(X1_SQL_DICT, X1_CSV_TEMP_STORAGE_PATH, X1_MYSQLDB, X1_MYSQLCURSOR)
extract_func.mysql_dict_sql_to_csv(SHIPPING_SQL_DICT, SHIPPING_CSV_TEMP_STORAGE_PATH, SHIPPING_MYSQLDB, SHIPPING_MYSQLCURSOR)
def x1_load_to_cloudstorage(X1_CSV_TEMP_STORAGE_PATH):
def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
#Config file name
CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
#Connect to cloudstorage
x1_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
shipping_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
#Get the filename and path in temp csv storage
temp_file_list = []
for (dirpath, dirnames, filenames) in walk(X1_CSV_TEMP_STORAGE_PATH):
for (dirpath, dirnames, filenames) in walk(SHIPPING_CSV_TEMP_STORAGE_PATH):
temp_file_list.extend(filenames)
#Loop all the file and upload to cloudstorage
for file_name in temp_file_list:
file_path = os.path.join(dirpath,file_name)
load_func.upload_to_bucket(x1_storage_client, file_name, file_path)
load_func.upload_to_bucket(shipping_storage_client, file_name, file_path)
def main():
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
#Make sure cron in the file directory
general_function.change_to_current_directory()
# #Extract Load Exchange Data
# EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
# exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
#Extract Load Exchange Data
EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# #Delete Temp CSV
# general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
# time.sleep(5)
#Delete Temp CSV
general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
#Make sure cron in the file directory
general_function.change_to_current_directory()
#Extract Load X1 Data
X1_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
x1_extract(X1_CSV_TEMP_STORAGE_PATH)
x1_load_to_cloudstorage(X1_CSV_TEMP_STORAGE_PATH)
#Extract Load SHIPPING Data
SHIPPING_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
general_function.delete_csv_in_path(X1_CSV_TEMP_STORAGE_PATH)
general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)