Add shipping ExtractLoad

shipping/extract/shipping_extract_function.py

@@ -0,0 +1,52 @@
+import csv
+from operator import concat
+import mysql.connector
+import os
+
+
+def connect_mysql(config):
+    try:  
+        mydb = mysql.connector.connect(
+        host      = config["host"],
+        user      = config["user"],
+        password  = config["password"],
+        database  = config["database"])
+        return mydb
+    except Exception as e:
+        print(e)
+        print("[FUNCTION_ERROR]-connect_mysql")
+        return False
+
+
+def mysql_dict_sql_to_csv(SQL_DICT, path, mysqldb, mysqlcursor):
+    temp_folder_path = path
+    # Check whether the specified path exists or not
+    temp_folder_path_isExist = os.path.exists(temp_folder_path)
+    if not temp_folder_path_isExist:
+        # Create a new directory because it does not exist 
+        os.makedirs(temp_folder_path)
+        print("[Create Folder] shipping_temp_extract_csv")
+
+    for tablename,sql in SQL_DICT.items():
+
+        try:
+            mysqlcursor.execute(sql)
+            mysqlresult = mysqlcursor.fetchall()
+            mysqldbname = mysqldb.database
+
+            filename = '/{}_{}.csv'.format(mysqldbname, tablename)
+            headers = [col[0] for col in mysqlcursor.description] # get headers
+            mysqlresult.insert(0, tuple(headers))
+            fp = open(concat(path,filename), 'w', newline = '', encoding = 'utf-8-sig') 
+            myFile = csv.writer(fp,delimiter='|',quotechar='"', quoting=csv.QUOTE_MINIMAL)
+            myFile.writerows(mysqlresult)
+            fp.close()
+
+        except Exception as e:
+            print(e)
+            print("[FUNCTION_ERROR]-write_to_csv:" , filename)
+            return False
+
+        finally:
+            print("[Download Complete]",filename)
+

shipping/load/shipping_load_function.py

@@ -0,0 +1,21 @@
+from google.cloud import storage
+
+def login_cloudstorage_credential(credential):
+    try:
+        storage_client = storage.Client.from_service_account_json(credential)
+        return storage_client
+    except Exception as e: 
+        print(e)
+        print("[FUNCTION_ERROR]-login_cloudstorage_credential")
+        return False
+
+def upload_to_bucket(storage_client, blob_name, file_path):
+    try:
+        exchange_bucket = storage_client.get_bucket('shipping_bucket_raw')
+        blob = exchange_bucket.blob(blob_name)
+        blob.upload_from_filename(file_path)
+        return print("[Upload Complete]",file_path)
+    except Exception as e: 
+        print(e)
+        print("[FUNCTION_ERROR]-upload_to_bucket",blob_name)
+        return False