cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 12:23:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add X1

Browse Source
This commit is contained in:
Yam ZhengLim
2022-11-12 12:30:42 +08:00
parent 12eed9fad4
commit acff209f06
4 changed files with 152 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
EXCHANGE_EXTRACT_SQL_OLD.json
+55
View File
@@ -20,3 +20,58 @@
}
{
"activity_log" : "SELECT * FROM activity_log;",
"addresses" : "SELECT * FROM addresses;",
"announcements" : "SELECT * FROM announcements;",
"announcement_segment" : "SELECT * FROM announcement_segment;",
// "banks" : "SELECT * FROM banks;",
// "bank_logs" : "SELECT * FROM bank_logs;",
// "bookings" : "SELECT * FROM bookings;",
"companies" : "SELECT * FROM companies;",
--"company_connections" : "SELECT * FROM company_connections;",
--"company_employees" : "SELECT * FROM company_employees;",
--"company_modules" : "SELECT * FROM company_modules;",
--"connection_segments" : "SELECT * FROM connection_segments;",
"contacts" : "SELECT * FROM contacts;",
--"containers" : "SELECT * FROM countries",
--"container_packing_lists" : "SELECT * FROM container_packing_lists",
"countries" : "SELECT * FROM countries;",
"currencies" : "SELECT * FROM currencies;",
// "currency_logs" : "SELECT * FROM currency_logs;",
// "currency_rates" : "SELECT * FROM currency_rates;",
// "currency_rate_logs" : "SELECT * FROM currency_rate_logs;",
"districts" : "SELECT * FROM districts;",
"documents" : "SELECT * FROM documents;",
// "employees" : "SELECT * FROM employees;",
"failed_jobs" : "SELECT * FROM failed_jobs;",
"files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', ''),'|', ''),'\\\\','') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
// "groups" : "SELECT * FROM groups;",
// "group_transactions" : "SELECT * FROM group_transactions;",
"jobs" : "SELECT * FROM jobs;",
"migrations" : "SELECT * FROM migrations;",
// "model_has_roles" : "SELECT * FROM model_has_roles;",
// "notifications" : "SELECT * FROM notifications;",
--"orders" : "SELECT * FROM orders",
--"order_roles" : "SELECT * FROM order_roles",
--"packages" : "SELECT * FROM packages",
--"packing_lists" : "SELECT * FROM packing_lists",
"password_resets" : "SELECT * FROM password_resets;",
// "permissions" : "SELECT * FROM permissions;",
--"remarks" : "SELECT * FROM remarks",
// "roles" : "SELECT * FROM roles;",
// "role_has_permissions" : "SELECT * FROM role_has_permissions;",
--"schedules" : "SELECT * FROM schedules",
"segments" : "SELECT * FROM segments;",
// "segment_companies" : "SELECT * FROM segment_companies;",
"segment_constants" : "SELECT * FROM segment_constants;",
// "service_types" : "SELECT * FROM service_types;",
"states" : "SELECT * FROM states;",
--"steps" : "SELECT * FROM steps",
"transactions" : "SELECT * FROM transactions;",
"transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;",
--"transport_arrangements" : "SELECT * FROM transport_arrangements",
"users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
"user_email_verifications" : "SELECT * FROM user_email_verifications;",
// "wallets" : "SELECT * FROM wallets;"
}
config/X1_CONFIG.json
+6
View File
@@ -0,0 +1,6 @@
{
"host" : "34.87.185.41",
"user" : "data_extracter",
"password" : "XVzm3u8Ilemgxu5S",
"database" : "exchange_production"
}
config/X1_EXTRACT_SQL.json
+38
View File
@@ -0,0 +1,38 @@
{
"activity_log" : "SELECT * FROM activity_log;",
"addresses" : "SELECT * FROM addresses;",
"announcements" : "SELECT * FROM announcements;",
"announcement_segment" : "SELECT * FROM announcement_segment;",
"companies" : "SELECT * FROM companies;",
"company_connections" : "SELECT * FROM company_connections;",
"company_employees" : "SELECT * FROM company_employees;",
"company_modules" : "SELECT * FROM company_modules;",
"connection_segments" : "SELECT * FROM connection_segments;",
"contacts" : "SELECT * FROM contacts;",
"containers" : "SELECT * FROM countries",
"container_packing_lists" : "SELECT * FROM container_packing_lists",
"countries" : "SELECT * FROM countries;",
"currencies" : "SELECT * FROM currencies;",
"districts" : "SELECT * FROM districts;",
"documents" : "SELECT * FROM documents;",
"failed_jobs" : "SELECT * FROM failed_jobs;",
"files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', ''),'|', ''),'\\\\','') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
"jobs" : "SELECT * FROM jobs;",
"migrations" : "SELECT * FROM migrations;",
"orders" : "SELECT * FROM orders",
"order_roles" : "SELECT * FROM order_roles",
"packages" : "SELECT * FROM packages",
"packing_lists" : "SELECT * FROM packing_lists",
"password_resets" : "SELECT * FROM password_resets;",
"remarks" : "SELECT * FROM remarks",
"schedules" : "SELECT * FROM schedules",
"segments" : "SELECT * FROM segments;",
"segment_constants" : "SELECT * FROM segment_constants;",
"states" : "SELECT * FROM states;",
"steps" : "SELECT * FROM steps",
"transactions" : "SELECT * FROM transactions;",
"transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;",
"transport_arrangements" : "SELECT * FROM transport_arrangements",
"users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
"user_email_verifications" : "SELECT * FROM user_email_verifications;"
}
main.py
+53 -7
View File
@@ -1,8 +1,8 @@
import os
from os import walk
import general_function
import exchange.extract.exchange_extract_function as ex_extract_func
import exchange.load.exchange_load_function as ex_load_func
import exchange.extract.exchange_extract_function as extract_func
import exchange.load.exchange_load_function as load_func
def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
#Config file name
@@ -15,18 +15,18 @@ def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
EX_SQL_DICT = general_function.read_json_file("./config/",EX_SQL_FILE)
#Connect to mysql
EX_MYSQLDB = ex_extract_func.connect_mysql(EX_MYSQL_CONFIG)
EX_MYSQLDB = extract_func.connect_mysql(EX_MYSQL_CONFIG)
EX_MYSQLCURSOR = EX_MYSQLDB.cursor()
#Run EX_SQL_DICT to get all data from mysql in csv format
ex_extract_func.mysql_dict_sql_to_csv(EX_SQL_DICT, EXCHANGE_CSV_TEMP_STORAGE_PATH, EX_MYSQLDB, EX_MYSQLCURSOR)
extract_func.mysql_dict_sql_to_csv(EX_SQL_DICT, EXCHANGE_CSV_TEMP_STORAGE_PATH, EX_MYSQLDB, EX_MYSQLCURSOR)
def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH):
#Config file name
CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
#Connect to cloudstorage
ex_storage_client = ex_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
ex_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
#Get the filename and path in temp csv storage
temp_file_list = []
@@ -36,17 +36,63 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH):
#Loop all the file and upload to cloudstorage
for file_name in temp_file_list:
file_path = os.path.join(dirpath,file_name)
ex_load_func.upload_to_bucket(ex_storage_client, file_name, file_path)
load_func.upload_to_bucket(ex_storage_client, file_name, file_path)
def x1_extract(X1_CSV_TEMP_STORAGE_PATH):
#Config file name
X1_MYSQL_CONFIG_FILE = "X1_CONFIG.json"
X1_SQL_FILE = "X1_EXTRACT_SQL.json"
X1_CSV_TEMP_STORAGE_PATH = X1_CSV_TEMP_STORAGE_PATH
#Read config file
X1_MYSQL_CONFIG = general_function.read_json_file("./config/",X1_MYSQL_CONFIG_FILE)
X1_SQL_DICT = general_function.read_json_file("./config/",X1_SQL_FILE)
#Connect to mysql
X1_MYSQLDB = extract_func.connect_mysql(X1_MYSQL_CONFIG)
X1_MYSQLCURSOR = X1_MYSQLDB.cursor()
#Run EX_SQL_DICT to get all data from mysql in csv format
extract_func.mysql_dict_sql_to_csv(X1_SQL_DICT, X1_CSV_TEMP_STORAGE_PATH, X1_MYSQLDB, X1_MYSQLCURSOR)
def x1_load_to_cloudstorage(X1_CSV_TEMP_STORAGE_PATH):
#Config file name
CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
#Connect to cloudstorage
x1_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
#Get the filename and path in temp csv storage
temp_file_list = []
for (dirpath, dirnames, filenames) in walk(X1_CSV_TEMP_STORAGE_PATH):
temp_file_list.extend(filenames)
#Loop all the file and upload to cloudstorage
for file_name in temp_file_list:
file_path = os.path.join(dirpath,file_name)
load_func.upload_to_bucket(x1_storage_client, file_name, file_path)
def main():
#Make sure cron in the file directory
general_function.change_to_current_directory()
#Extract Load Exchange Data, and delete the temp csv
#Extract Load Exchange Data
EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
#Extract Load X1 Data
X1_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
x1_extract(X1_CSV_TEMP_STORAGE_PATH)
x1_load_to_cloudstorage(X1_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)