diff --git a/EXCHANGE_EXTRACT_SQL_OLD.json b/EXCHANGE_EXTRACT_SQL_OLD.json index 4c6c9a6..759c8f2 100644 --- a/EXCHANGE_EXTRACT_SQL_OLD.json +++ b/EXCHANGE_EXTRACT_SQL_OLD.json @@ -20,3 +20,58 @@ } +{ + "activity_log" : "SELECT * FROM activity_log;", + "addresses" : "SELECT * FROM addresses;", + "announcements" : "SELECT * FROM announcements;", + "announcement_segment" : "SELECT * FROM announcement_segment;", + // "banks" : "SELECT * FROM banks;", + // "bank_logs" : "SELECT * FROM bank_logs;", + // "bookings" : "SELECT * FROM bookings;", + "companies" : "SELECT * FROM companies;", + --"company_connections" : "SELECT * FROM company_connections;", + --"company_employees" : "SELECT * FROM company_employees;", + --"company_modules" : "SELECT * FROM company_modules;", + --"connection_segments" : "SELECT * FROM connection_segments;", + "contacts" : "SELECT * FROM contacts;", + --"containers" : "SELECT * FROM countries", + --"container_packing_lists" : "SELECT * FROM container_packing_lists", + "countries" : "SELECT * FROM countries;", + "currencies" : "SELECT * FROM currencies;", + // "currency_logs" : "SELECT * FROM currency_logs;", + // "currency_rates" : "SELECT * FROM currency_rates;", + // "currency_rate_logs" : "SELECT * FROM currency_rate_logs;", + "districts" : "SELECT * FROM districts;", + "documents" : "SELECT * FROM documents;", + // "employees" : "SELECT * FROM employees;", + "failed_jobs" : "SELECT * FROM failed_jobs;", + "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', ''),'|', ''),'\\\\','') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;", + // "groups" : "SELECT * FROM groups;", + // "group_transactions" : "SELECT * FROM group_transactions;", + "jobs" : "SELECT * FROM jobs;", + "migrations" : "SELECT * FROM migrations;", + // "model_has_roles" : "SELECT * FROM model_has_roles;", + // "notifications" : "SELECT * FROM notifications;", + --"orders" : "SELECT * FROM orders", + --"order_roles" : "SELECT * FROM order_roles", + --"packages" : "SELECT * FROM packages", + --"packing_lists" : "SELECT * FROM packing_lists", + "password_resets" : "SELECT * FROM password_resets;", + // "permissions" : "SELECT * FROM permissions;", + --"remarks" : "SELECT * FROM remarks", + // "roles" : "SELECT * FROM roles;", + // "role_has_permissions" : "SELECT * FROM role_has_permissions;", + --"schedules" : "SELECT * FROM schedules", + "segments" : "SELECT * FROM segments;", + // "segment_companies" : "SELECT * FROM segment_companies;", + "segment_constants" : "SELECT * FROM segment_constants;", + // "service_types" : "SELECT * FROM service_types;", + "states" : "SELECT * FROM states;", + --"steps" : "SELECT * FROM steps", + "transactions" : "SELECT * FROM transactions;", + "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;", + --"transport_arrangements" : "SELECT * FROM transport_arrangements", + "users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;", + "user_email_verifications" : "SELECT * FROM user_email_verifications;", + // "wallets" : "SELECT * FROM wallets;" +} \ No newline at end of file diff --git a/config/X1_CONFIG.json b/config/X1_CONFIG.json new file mode 100644 index 0000000..dab76a7 --- /dev/null +++ b/config/X1_CONFIG.json @@ -0,0 +1,6 @@ +{ + "host" : "34.87.185.41", + "user" : "data_extracter", + "password" : "XVzm3u8Ilemgxu5S", + "database" : "exchange_production" +} \ No newline at end of file diff --git a/config/X1_EXTRACT_SQL.json b/config/X1_EXTRACT_SQL.json new file mode 100644 index 0000000..fbe0e01 --- /dev/null +++ b/config/X1_EXTRACT_SQL.json @@ -0,0 +1,38 @@ +{ + "activity_log" : "SELECT * FROM activity_log;", + "addresses" : "SELECT * FROM addresses;", + "announcements" : "SELECT * FROM announcements;", + "announcement_segment" : "SELECT * FROM announcement_segment;", + "companies" : "SELECT * FROM companies;", + "company_connections" : "SELECT * FROM company_connections;", + "company_employees" : "SELECT * FROM company_employees;", + "company_modules" : "SELECT * FROM company_modules;", + "connection_segments" : "SELECT * FROM connection_segments;", + "contacts" : "SELECT * FROM contacts;", + "containers" : "SELECT * FROM countries", + "container_packing_lists" : "SELECT * FROM container_packing_lists", + "countries" : "SELECT * FROM countries;", + "currencies" : "SELECT * FROM currencies;", + "districts" : "SELECT * FROM districts;", + "documents" : "SELECT * FROM documents;", + "failed_jobs" : "SELECT * FROM failed_jobs;", + "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', ''),'|', ''),'\\\\','') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;", + "jobs" : "SELECT * FROM jobs;", + "migrations" : "SELECT * FROM migrations;", + "orders" : "SELECT * FROM orders", + "order_roles" : "SELECT * FROM order_roles", + "packages" : "SELECT * FROM packages", + "packing_lists" : "SELECT * FROM packing_lists", + "password_resets" : "SELECT * FROM password_resets;", + "remarks" : "SELECT * FROM remarks", + "schedules" : "SELECT * FROM schedules", + "segments" : "SELECT * FROM segments;", + "segment_constants" : "SELECT * FROM segment_constants;", + "states" : "SELECT * FROM states;", + "steps" : "SELECT * FROM steps", + "transactions" : "SELECT * FROM transactions;", + "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;", + "transport_arrangements" : "SELECT * FROM transport_arrangements", + "users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;", + "user_email_verifications" : "SELECT * FROM user_email_verifications;" +} diff --git a/main.py b/main.py index bc24de0..6bf4eb9 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,8 @@ import os from os import walk import general_function -import exchange.extract.exchange_extract_function as ex_extract_func -import exchange.load.exchange_load_function as ex_load_func +import exchange.extract.exchange_extract_function as extract_func +import exchange.load.exchange_load_function as load_func def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH): #Config file name @@ -15,18 +15,18 @@ def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH): EX_SQL_DICT = general_function.read_json_file("./config/",EX_SQL_FILE) #Connect to mysql - EX_MYSQLDB = ex_extract_func.connect_mysql(EX_MYSQL_CONFIG) + EX_MYSQLDB = extract_func.connect_mysql(EX_MYSQL_CONFIG) EX_MYSQLCURSOR = EX_MYSQLDB.cursor() #Run EX_SQL_DICT to get all data from mysql in csv format - ex_extract_func.mysql_dict_sql_to_csv(EX_SQL_DICT, EXCHANGE_CSV_TEMP_STORAGE_PATH, EX_MYSQLDB, EX_MYSQLCURSOR) + extract_func.mysql_dict_sql_to_csv(EX_SQL_DICT, EXCHANGE_CSV_TEMP_STORAGE_PATH, EX_MYSQLDB, EX_MYSQLCURSOR) def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH): #Config file name CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json" #Connect to cloudstorage - ex_storage_client = ex_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE) + ex_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE) #Get the filename and path in temp csv storage temp_file_list = [] @@ -36,17 +36,63 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH): #Loop all the file and upload to cloudstorage for file_name in temp_file_list: file_path = os.path.join(dirpath,file_name) - ex_load_func.upload_to_bucket(ex_storage_client, file_name, file_path) + load_func.upload_to_bucket(ex_storage_client, file_name, file_path) + + + +def x1_extract(X1_CSV_TEMP_STORAGE_PATH): + #Config file name + X1_MYSQL_CONFIG_FILE = "X1_CONFIG.json" + X1_SQL_FILE = "X1_EXTRACT_SQL.json" + X1_CSV_TEMP_STORAGE_PATH = X1_CSV_TEMP_STORAGE_PATH + + #Read config file + X1_MYSQL_CONFIG = general_function.read_json_file("./config/",X1_MYSQL_CONFIG_FILE) + X1_SQL_DICT = general_function.read_json_file("./config/",X1_SQL_FILE) + + #Connect to mysql + X1_MYSQLDB = extract_func.connect_mysql(X1_MYSQL_CONFIG) + X1_MYSQLCURSOR = X1_MYSQLDB.cursor() + + #Run EX_SQL_DICT to get all data from mysql in csv format + extract_func.mysql_dict_sql_to_csv(X1_SQL_DICT, X1_CSV_TEMP_STORAGE_PATH, X1_MYSQLDB, X1_MYSQLCURSOR) + + +def x1_load_to_cloudstorage(X1_CSV_TEMP_STORAGE_PATH): + #Config file name + CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json" + + #Connect to cloudstorage + x1_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE) + + #Get the filename and path in temp csv storage + temp_file_list = [] + for (dirpath, dirnames, filenames) in walk(X1_CSV_TEMP_STORAGE_PATH): + temp_file_list.extend(filenames) + + #Loop all the file and upload to cloudstorage + for file_name in temp_file_list: + file_path = os.path.join(dirpath,file_name) + load_func.upload_to_bucket(x1_storage_client, file_name, file_path) + + def main(): #Make sure cron in the file directory general_function.change_to_current_directory() - #Extract Load Exchange Data, and delete the temp csv + #Extract Load Exchange Data EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/" exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH) exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH) + + #Extract Load X1 Data + X1_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/" + x1_extract(X1_CSV_TEMP_STORAGE_PATH) + x1_load_to_cloudstorage(X1_CSV_TEMP_STORAGE_PATH) + + #Delete Temp CSV general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)