From e940b729e94d40c24d5323bff97a42c5fe890bc6 Mon Sep 17 00:00:00 2001
From: Yam ZhengLim <yamzhenglim0427@gmail.com>
Date: Sun, 13 Nov 2022 21:06:35 +0800
Subject: [PATCH] Add shipping ExtractLoad

---
 config/EXCHANGE_EXTRACT_SQL.json              |   4 +-
 config/SHIPPING_EXTRACT_SQL.json              |   4 +-
 main.py                                       |  24 ++++----
 .../exchange_extract_function.cpython-39.pyc  | Bin 0 -> 1940 bytes
 shipping/extract/shipping_extract_function.py |  52 ++++++++++++++++++
 .../shipping_temp_extract_csv/.gitkeep        |   0
 .../exchange_load_function.cpython-39.pyc     | Bin 0 -> 985 bytes
 shipping/load/shipping_load_function.py       |  21 +++++++
 8 files changed, 90 insertions(+), 15 deletions(-)
 create mode 100644 shipping/extract/__pycache__/exchange_extract_function.cpython-39.pyc
 create mode 100644 shipping/extract/shipping_extract_function.py
 create mode 100644 shipping/extract/shipping_temp_extract_csv/.gitkeep
 create mode 100644 shipping/load/__pycache__/exchange_load_function.cpython-39.pyc
 create mode 100644 shipping/load/shipping_load_function.py

diff --git a/config/EXCHANGE_EXTRACT_SQL.json b/config/EXCHANGE_EXTRACT_SQL.json
index 8d6eeb2..f39a497 100644
--- a/config/EXCHANGE_EXTRACT_SQL.json
+++ b/config/EXCHANGE_EXTRACT_SQL.json
@@ -17,7 +17,7 @@
     "documents"                 :   "SELECT * FROM documents;",
     "employees"                 :   "SELECT * FROM employees;",
     "failed_jobs"               :   "SELECT * FROM failed_jobs;",
-    "files"                     :   "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', ''),'|', ''),'\\\\','') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
+    "files"                     :   "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '<br>'),'|', '&#124'),'\\\\','&#92&#92') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
     "groups"                    :   "SELECT * FROM groups;",  
     "group_transactions"        :   "SELECT * FROM group_transactions;", 
     "jobs"                      :   "SELECT * FROM jobs;", 
@@ -34,7 +34,7 @@
     "service_types"             :   "SELECT * FROM service_types;",
     "states"                    :   "SELECT * FROM states;",
     "transactions"              :   "SELECT * FROM transactions;",
-    "transaction_detail"        :   "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at  FROM `transaction_detail`;",
+    "transaction_detail"        :   "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', '<br>'),'|', '&#124'),'\\\\','&#92&#92') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', '<br>'),'|', '&#124'),'\\\\','&#92&#92') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at  FROM `transaction_detail`;",
     "users"                     :   "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
     "user_email_verifications"  :   "SELECT * FROM user_email_verifications;",
     "wallets"                   :   "SELECT * FROM wallets;"
diff --git a/config/SHIPPING_EXTRACT_SQL.json b/config/SHIPPING_EXTRACT_SQL.json
index 85f5483..c6dc987 100644
--- a/config/SHIPPING_EXTRACT_SQL.json
+++ b/config/SHIPPING_EXTRACT_SQL.json
@@ -16,7 +16,7 @@
     "districts"                 :   "SELECT * FROM districts;",
     "documents"                 :   "SELECT * FROM documents;",
     "failed_jobs"               :   "SELECT * FROM failed_jobs;",
-    "files"                     :   "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', ''),'|', ''),'\\\\','') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
+    "files"                     :   "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '<br>'),'|', '&#124'),'\\\\','&#92&#92') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
     "jobs"                      :   "SELECT * FROM jobs;", 
     "migrations"                :   "SELECT * FROM migrations;",
     "orders"                    :   "SELECT * FROM orders",
@@ -24,7 +24,7 @@
     "packages"                  :   "SELECT * FROM packages",    
     "packing_lists"             :   "SELECT * FROM packing_lists",    
     "password_resets"           :   "SELECT * FROM password_resets;", 
-    "remarks"                   :   "SELECT id, owner_type, owner_id, commenter_id, REPLACE(REPLACE(REPLACE(content, '\n', ''),'|', ''),'\\\\','') AS 'content',, deleted_at, created_at, updated_at FROM remarks",
+    "remarks"                   :   "SELECT id, owner_type, owner_id, commenter_id, REPLACE(REPLACE(REPLACE(content, '\n', '<br>'),'|', '&#124'),'\\\\','&#92&#92') AS 'content',, deleted_at, created_at, updated_at FROM remarks",
     "schedules"                 :   "SELECT * FROM schedules",
     "segments"                  :   "SELECT * FROM segments;",
     "segment_constants"         :   "SELECT * FROM segment_constants;",
diff --git a/main.py b/main.py
index 216b9aa..ca61cb5 100644
--- a/main.py
+++ b/main.py
@@ -1,8 +1,10 @@
 import os
 from os import walk
 import general_function
-import exchange.extract.exchange_extract_function as extract_func
-import exchange.load.exchange_load_function as load_func
+import exchange.extract.exchange_extract_function as exchange_extract_func
+import exchange.load.exchange_load_function as exchange_load_func
+import shipping.extract.shipping_extract_function as shipping_extract_func
+import shipping.load.shipping_load_function as shipping_load_func
 
 def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
     #Config file name
@@ -15,18 +17,18 @@ def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
     EX_SQL_DICT = general_function.read_json_file("./config/",EX_SQL_FILE)
 
     #Connect to mysql
-    EX_MYSQLDB = extract_func.connect_mysql(EX_MYSQL_CONFIG)
+    EX_MYSQLDB = exchange_extract_func.connect_mysql(EX_MYSQL_CONFIG)
     EX_MYSQLCURSOR = EX_MYSQLDB.cursor()
 
     #Run EX_SQL_DICT to get all data from mysql in csv format
-    extract_func.mysql_dict_sql_to_csv(EX_SQL_DICT, EXCHANGE_CSV_TEMP_STORAGE_PATH, EX_MYSQLDB, EX_MYSQLCURSOR)
+    exchange_extract_func.mysql_dict_sql_to_csv(EX_SQL_DICT, EXCHANGE_CSV_TEMP_STORAGE_PATH, EX_MYSQLDB, EX_MYSQLCURSOR)
 
 def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH):
     #Config file name
     CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
 
     #Connect to cloudstorage
-    ex_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
+    ex_storage_client = exchange_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
 
     #Get the filename and path in temp csv storage
     temp_file_list = []
@@ -36,7 +38,7 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH):
     #Loop all the file and upload to cloudstorage
     for file_name in temp_file_list:
         file_path = os.path.join(dirpath,file_name)
-        load_func.upload_to_bucket(ex_storage_client, file_name, file_path)
+        exchange_load_func.upload_to_bucket(ex_storage_client, file_name, file_path)
 
 
 
@@ -51,11 +53,11 @@ def shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH):
     SHIPPING_SQL_DICT = general_function.read_json_file("./config/",SHIPPING_SQL_FILE)
 
     #Connect to mysql
-    SHIPPING_MYSQLDB = extract_func.connect_mysql(SHIPPING_MYSQL_CONFIG)
+    SHIPPING_MYSQLDB = shipping_extract_func.connect_mysql(SHIPPING_MYSQL_CONFIG)
     SHIPPING_MYSQLCURSOR = SHIPPING_MYSQLDB.cursor()
 
     #Run EX_SQL_DICT to get all data from mysql in csv format
-    extract_func.mysql_dict_sql_to_csv(SHIPPING_SQL_DICT, SHIPPING_CSV_TEMP_STORAGE_PATH, SHIPPING_MYSQLDB, SHIPPING_MYSQLCURSOR)
+    shipping_extract_func.mysql_dict_sql_to_csv(SHIPPING_SQL_DICT, SHIPPING_CSV_TEMP_STORAGE_PATH, SHIPPING_MYSQLDB, SHIPPING_MYSQLCURSOR)
 
 
 def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
@@ -63,7 +65,7 @@ def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
     CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
 
     #Connect to cloudstorage
-    shipping_storage_client = load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
+    shipping_storage_client = shipping_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
 
     #Get the filename and path in temp csv storage
     temp_file_list = []
@@ -73,7 +75,7 @@ def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
     #Loop all the file and upload to cloudstorage
     for file_name in temp_file_list:
         file_path = os.path.join(dirpath,file_name)
-        load_func.upload_to_bucket(shipping_storage_client, file_name, file_path)
+        shipping_load_func.upload_to_bucket(shipping_storage_client, file_name, file_path)
 
 
 
@@ -94,7 +96,7 @@ def main():
     general_function.change_to_current_directory()
 
     #Extract Load SHIPPING Data
-    SHIPPING_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/"
+    SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/"
     shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH)
     shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH)
 
diff --git a/shipping/extract/__pycache__/exchange_extract_function.cpython-39.pyc b/shipping/extract/__pycache__/exchange_extract_function.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fadded5f6046aba33f1ba19316aaea9ca8faff0c
GIT binary patch
literal 1940
zcmZuyOK%%D5GJ`#y=>WeH0~kDp-mwJR@_S)1cp&Svd{pngW3s<Y-FgLC1tO(51FK5
zOWDoE?J;Qj52Ry%NB=@ky!O;<dox;~!<FnhX_nw{IDE_u=bKr-Rx2a;)_?sn_|ZY=
zcRjiK5tw`hUEBvk5ycVm^hhEc6OZ7T%<#-&&GIa2Q1b`m+0>#oj1I-SsNwtpr_pu;
zD?3P&z?T8mg865_sI6#07kwb(Rf&%95y?Q`3Ia+BY82F5LDYI<<~YN9Xcw7C8l&$q
z%E6ZLZE0rY3}YlsYU^<iJp~JM@|sX*ch0S&b>P4%cUzf<sgTMX3dWT?@P#-@IaMz8
zrT^R)tYLC(95*$wG_kdLx1V*MY_}die6-!$+Sz%uv%dyjB`lD=cqERaPQy}G!Fi?m
zLdrd7&Z^SZX}|_DOcP}dc$mnB;pu1ThX<Y+kLYuS*)ND!K-2kg>p}Mk7$>?TKOP@;
zA0@2K!<VevX5xiR2i?}gtxm5E=6>rZ{sD_wBD>o9wfMg|I}Hx~<bZYAspNhjVVnti
zb3v~^Oag6Cb1+i1cYFa%AaM%_!UWqF?Qh$%@d9?C@fxfJ`hhOKWni@NMK~uA=nUtk
z4(Nnr<`m~vW{JBJO$<FNLWOmxGBmNKsEyv@{i80)?c8~b$2;Tm%$V4jqjv)5Oi}J;
z<jBoSluSxlDYK^}FV8@i8As(dI>UJ-b0-yQ$!b<P!&3l;tjUE8XXP^jupmEqdG(6K
z0m*7!%c_&bDVo%&OG|L)LROnBWeZyG9>B$fc4lCd)IUe!w{?Vz_7v#<R{y-ZdQ}fC
z(+aHubQk6F(Y0(5G}Y+B8zZl0^~v?jEqq*@+?W=(CSL=zZ@$H&*8uJ%0Cy9>y)*~+
zqW0wy-b=5S_B7<{GbtqHQUP|5Yp`EkJBC{>_N}AiW|0WX+vCQw7H7U>cRFcA8Q;Hi
zi4n=-!6i@v@p4?g_bTtb%A3Hylg7GKXdLW97uSItKuq_mI#DTFK|hmY6Os;do%#j3
z8$^-jx@QDwRLp0j3d^dz5rskqX*}4x3-iCAqj96isU8g>(ICxwGVOs4&5bAx{77s9
zb8)87p)gnY<Q!+`_?|-J(opu-?yrgP0Q5QuJBh-CDVHTdO2g#f9DmjzUWu|Oj6=z|
zS2`Z1l0l;I59Ip^hdpxx9*nO&Yo{kkl=}2e3rvofWc!`QM~b9EnFC)QDw~~xZ9=)R
z|ANtw3trN0T5wA&JcpgKU?`dA_L&5YQB>3!W%pAa`_ijWCITMLYK9#qP!OcD<PbC|
zGaaym>$g`XM35?nvq9tsOj(7I%78agcEN*FvoSpp$_k<sE^^^yW7%_ezx$@we%N}<
zKL&y46i-c`dzFF-hFm}is~1@47YONTzr0)TOM0Q$(yn?XsGt!`{Fr$L1Z#c~XJQzE
zL-XtPvU2<3e4}#+4nVQ<z(+0goD_nl8B^;~q(AWNc+>$2*FTK+QFG&i0J#A>#TP&j
zHc1tF8M|be*ce7^gQO1a6X+d$6KEM%NgY;TG|K-h8+;W`HeCKG49bPq=DtjM@#5ZM
up|E*b<hd@Fd_@yo7K@`-jMP<;7x;EJ;*<^}wyBdrXkT1h)%DbH7yboz%>Ug0

literal 0
HcmV?d00001

diff --git a/shipping/extract/shipping_extract_function.py b/shipping/extract/shipping_extract_function.py
new file mode 100644
index 0000000..2fc5dcb
--- /dev/null
+++ b/shipping/extract/shipping_extract_function.py
@@ -0,0 +1,52 @@
+import csv
+from operator import concat
+import mysql.connector
+import os
+
+
+def connect_mysql(config):
+    try:  
+        mydb = mysql.connector.connect(
+        host      = config["host"],
+        user      = config["user"],
+        password  = config["password"],
+        database  = config["database"])
+        return mydb
+    except Exception as e:
+        print(e)
+        print("[FUNCTION_ERROR]-connect_mysql")
+        return False
+
+
+def mysql_dict_sql_to_csv(SQL_DICT, path, mysqldb, mysqlcursor):
+    temp_folder_path = path
+    # Check whether the specified path exists or not
+    temp_folder_path_isExist = os.path.exists(temp_folder_path)
+    if not temp_folder_path_isExist:
+        # Create a new directory because it does not exist 
+        os.makedirs(temp_folder_path)
+        print("[Create Folder] shipping_temp_extract_csv")
+
+    for tablename,sql in SQL_DICT.items():
+
+        try:
+            mysqlcursor.execute(sql)
+            mysqlresult = mysqlcursor.fetchall()
+            mysqldbname = mysqldb.database
+
+            filename = '/{}_{}.csv'.format(mysqldbname, tablename)
+            headers = [col[0] for col in mysqlcursor.description] # get headers
+            mysqlresult.insert(0, tuple(headers))
+            fp = open(concat(path,filename), 'w', newline = '', encoding = 'utf-8-sig') 
+            myFile = csv.writer(fp,delimiter='|',quotechar='"', quoting=csv.QUOTE_MINIMAL)
+            myFile.writerows(mysqlresult)
+            fp.close()
+
+        except Exception as e:
+            print(e)
+            print("[FUNCTION_ERROR]-write_to_csv:" , filename)
+            return False
+
+        finally:
+            print("[Download Complete]",filename)
+
diff --git a/shipping/extract/shipping_temp_extract_csv/.gitkeep b/shipping/extract/shipping_temp_extract_csv/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/shipping/load/__pycache__/exchange_load_function.cpython-39.pyc b/shipping/load/__pycache__/exchange_load_function.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5aebbb1bdb19b47013547e82dfcc7985b74403bc
GIT binary patch
literal 985
zcmaKrPfy!06u@mKDQ#8Q#&+3Z6FYVih}$GIO_NHiaafUH2rUFzZfx2>5=V~9Xq0m4
zJ_sE5A@&9K5pvyWUtzoK*_48qG)J-DdwzL;{BzjyaudP$@%>HshamLRDE9~0(0h3E
zHJCW!I7SKfF-Dwl=PU9Fcew|v!)yDfz4QybP#afExt59zMTm`IJ{(w0whp{`AEshe
zqf>lEN_35<xWtEOAGt_7lMfh82}Z?B?UrN)*|f{CSr5@?sA5Vk2`Bq=X%lUN1lgT_
zv2oNp=yyKt?DlDQZ*O<+cs-WGD5YU6$9!HB4VB;`)e(z(ZBLmF{aPoEz~nzaQ8J;q
zQ0Gx7C<{Y5PBlHvWm+}57oo^>1frLzDAjGpZ{CHfRu<uHs0xd3NG(C|-)Ehz-~eLg
z!GtBnDA-NKwu;V0ur2a4EwiAr)9ul1rrAf9!l5NlNx(r}PwsSa5sp|o6hSN*4{kMT
zG<q^lLsQsBHmP3xw_$@IUx5i>f?K$TS8yF3)c{LqHg87_gaxY+D8=g%T>*a}<&@49
z&;rRVU7$y<4IzhvG$V6{?qmQAJz%(|ozk-m9qyEP23+S9y-Sa~mLf<CieBG;x@d1O
z4$p+9ihU`b935n)+qI5NvRG(wT)g^+_Sm{mE$Pjd0N8I1A;OJXxr0~^{AY7F3;QIB
zMamMPT<B7nGicY82aDe@&eZHx4dX>Krbqsx2gNCKOv;>>rF}EiL9HG?tfmFld=(~h
z`c3<^EP*CgX0P9N{nk**VJtT6jkTB4!m<1J5}1pnY?k`lMDlSg-kD!5H%+uiz3~^Q
C`~DCB

literal 0
HcmV?d00001

diff --git a/shipping/load/shipping_load_function.py b/shipping/load/shipping_load_function.py
new file mode 100644
index 0000000..3d32ec7
--- /dev/null
+++ b/shipping/load/shipping_load_function.py
@@ -0,0 +1,21 @@
+from google.cloud import storage
+
+def login_cloudstorage_credential(credential):
+    try:
+        storage_client = storage.Client.from_service_account_json(credential)
+        return storage_client
+    except Exception as e: 
+        print(e)
+        print("[FUNCTION_ERROR]-login_cloudstorage_credential")
+        return False
+
+def upload_to_bucket(storage_client, blob_name, file_path):
+    try:
+        exchange_bucket = storage_client.get_bucket('shipping_bucket_raw')
+        blob = exchange_bucket.blob(blob_name)
+        blob.upload_from_filename(file_path)
+        return print("[Upload Complete]",file_path)
+    except Exception as e: 
+        print(e)
+        print("[FUNCTION_ERROR]-upload_to_bucket",blob_name)
+        return False
\ No newline at end of file