From 3f25d3debcd6d2b7424adb705815d35e56b458d2 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Sun, 13 Nov 2022 18:00:42 +0800 Subject: [PATCH] Change X1 to Shipping --- .../{X1_CONFIG.json => SHIPPING_CONFIG.json} | 0 ...ACT_SQL.json => SHIPPING_EXTRACT_SQL.json} | 2 +- main.py | 54 +++++++++---------- 3 files changed, 27 insertions(+), 29 deletions(-) rename config/{X1_CONFIG.json => SHIPPING_CONFIG.json} (100%) rename config/{X1_EXTRACT_SQL.json => SHIPPING_EXTRACT_SQL.json} (92%) diff --git a/config/X1_CONFIG.json b/config/SHIPPING_CONFIG.json similarity index 100% rename from config/X1_CONFIG.json rename to config/SHIPPING_CONFIG.json diff --git a/config/X1_EXTRACT_SQL.json b/config/SHIPPING_EXTRACT_SQL.json similarity index 92% rename from config/X1_EXTRACT_SQL.json rename to config/SHIPPING_EXTRACT_SQL.json index 53072ca..85f5483 100644 --- a/config/X1_EXTRACT_SQL.json +++ b/config/SHIPPING_EXTRACT_SQL.json @@ -24,7 +24,7 @@ "packages" : "SELECT * FROM packages", "packing_lists" : "SELECT * FROM packing_lists", "password_resets" : "SELECT * FROM password_resets;", - "remarks" : "SELECT * FROM remarks", + "remarks" : "SELECT id, owner_type, owner_id, commenter_id, REPLACE(REPLACE(REPLACE(content, '\n', ''),'|', ''),'\\\\','') AS 'content',, deleted_at, created_at, updated_at FROM remarks", "schedules" : "SELECT * FROM schedules", "segments" : "SELECT * FROM segments;", "segment_constants" : "SELECT * FROM segment_constants;", diff --git a/main.py b/main.py index 797be26..216b9aa 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,5 @@ import os from os import walk -import time import general_function import exchange.extract.exchange_extract_function as extract_func import exchange.load.exchange_load_function as load_func @@ -41,67 +40,66 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH): -def x1_extract(X1_CSV_TEMP_STORAGE_PATH): +def shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH): #Config file name - X1_MYSQL_CONFIG_FILE = "X1_CONFIG.json" - X1_SQL_FILE = "X1_EXTRACT_SQL.json" - X1_CSV_TEMP_STORAGE_PATH = X1_CSV_TEMP_STORAGE_PATH + SHIPPING_MYSQL_CONFIG_FILE = "SHIPPING_CONFIG.json" + SHIPPING_SQL_FILE = "SHIPPING_EXTRACT_SQL.json" + SHIPPING_CSV_TEMP_STORAGE_PATH = SHIPPING_CSV_TEMP_STORAGE_PATH #Read config file - X1_MYSQL_CONFIG = general_function.read_json_file("./config/",X1_MYSQL_CONFIG_FILE) - X1_SQL_DICT = general_function.read_json_file("./config/",X1_SQL_FILE) + SHIPPING_MYSQL_CONFIG = general_function.read_json_file("./config/",SHIPPING_MYSQL_CONFIG_FILE) + SHIPPING_SQL_DICT = general_function.read_json_file("./config/",SHIPPING_SQL_FILE) #Connect to mysql - X1_MYSQLDB = extract_func.connect_mysql(X1_MYSQL_CONFIG) - X1_MYSQLCURSOR = X1_MYSQLDB.cursor() + SHIPPING_MYSQLDB = extract_func.connect_mysql(SHIPPING_MYSQL_CONFIG) + SHIPPING_MYSQLCURSOR = SHIPPING_MYSQLDB.cursor() #Run EX_SQL_DICT to get all data from mysql in csv format - extract_func.mysql_dict_sql_to_csv(X1_SQL_DICT, X1_CSV_TEMP_STORAGE_PATH, X1_MYSQLDB, X1_MYSQLCURSOR) + extract_func.mysql_dict_sql_to_csv(SHIPPING_SQL_DICT, SHIPPING_CSV_TEMP_STORAGE_PATH, SHIPPING_MYSQLDB, SHIPPING_MYSQLCURSOR) -def x1_load_to_cloudstorage(X1_CSV_TEMP_STORAGE_PATH): +def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH): #Config file name CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json" #Connect to cloudstorage - x1_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE) + shipping_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE) #Get the filename and path in temp csv storage temp_file_list = [] - for (dirpath, dirnames, filenames) in walk(X1_CSV_TEMP_STORAGE_PATH): + for (dirpath, dirnames, filenames) in walk(SHIPPING_CSV_TEMP_STORAGE_PATH): temp_file_list.extend(filenames) #Loop all the file and upload to cloudstorage for file_name in temp_file_list: file_path = os.path.join(dirpath,file_name) - load_func.upload_to_bucket(x1_storage_client, file_name, file_path) + load_func.upload_to_bucket(shipping_storage_client, file_name, file_path) def main(): - # #Make sure cron in the file directory - # general_function.change_to_current_directory() + #Make sure cron in the file directory + general_function.change_to_current_directory() - # #Extract Load Exchange Data - # EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/" - # exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH) - # exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH) + #Extract Load Exchange Data + EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/" + exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH) + exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH) - # #Delete Temp CSV - # general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH) - # time.sleep(5) + #Delete Temp CSV + general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH) #Make sure cron in the file directory general_function.change_to_current_directory() - #Extract Load X1 Data - X1_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/" - x1_extract(X1_CSV_TEMP_STORAGE_PATH) - x1_load_to_cloudstorage(X1_CSV_TEMP_STORAGE_PATH) + #Extract Load SHIPPING Data + SHIPPING_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/" + shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH) + shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH) #Delete Temp CSV - general_function.delete_csv_in_path(X1_CSV_TEMP_STORAGE_PATH) + general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)