Exchange Extract Load Function

exchange/extract/exchange_extract_function.py

@@ -0,0 +1,45 @@
+import json
+import csv
+from operator import concat
+import mysql.connector
+
+
+def connect_mysql(config):
+    try:  
+        mydb = mysql.connector.connect(
+        host      = config["host"],
+        user      = config["user"],
+        password  = config["password"],
+        database  = config["database"])
+        return mydb
+    except Exception as e:
+        print(e)
+        print("[FUNCTION_ERROR]-read_json_file:" , name)
+        return False
+
+
+def mysql_dict_sql_to_csv(SQL_DICT, path, mysqldb, mysqlcursor):
+
+    for tablename,sql in SQL_DICT.items():
+
+        try:
+            mysqlcursor.execute(sql)
+            mysqlresult = mysqlcursor.fetchall()
+            mysqldbname = mysqldb.database
+
+            filename = '/{}_{}.csv'.format(mysqldbname, tablename)
+            headers = [col[0] for col in mysqlcursor.description] # get headers
+            mysqlresult.insert(0, tuple(headers))
+            fp = open(concat(path,filename), 'w', newline = '') 
+            myFile = csv.writer(fp)
+            myFile.writerows(mysqlresult)
+            fp.close()
+
+        except Exception as e:
+            print(e)
+            print("[FUNCTION_ERROR]-write_to_csv:" , filename)
+            return False
+
+        finally:
+            print("[Download Complete]",filename)
+

exchange/load/exchange_load_function.py

@@ -0,0 +1,23 @@
+import os
+from google.cloud import storage
+import json
+
+def login_cloudstorage_credential(credential):
+    try:
+        storage_client = storage.Client.from_service_account_json(credential)
+        return storage_client
+    except Exception as e: 
+        print(e)
+        print("[FUNCTION_ERROR]-login_cloudstorage_credential")
+        return False
+
+def upload_to_bucket(storage_client, blob_name, file_path):
+    try:
+        exchange_bucket = storage_client.get_bucket('exchange_bucket_raw')
+        blob = exchange_bucket.blob(blob_name)
+        blob.upload_from_filename(file_path)
+        return print("[Upload Complete]",file_path)
+    except Exception as e: 
+        print(e)
+        print("[FUNCTION_ERROR]-upload_to_bucket",blob_name)
+        return False