Exchange Extract,Load, and delete temp

.gitignore

@@ -1,2 +1,3 @@
 *.csv
-__pycache__
+__pycache__
+testing_sql.txt

README.md

@@ -1,17 +1,11 @@
-# Guide
+# CIEF Exchange Extract Load
 
- "customers"    :   "SELECT * FROM customers;",
- "employees"    :   "SELECT * FROM employees;",
- "offices"      :   "SELECT * FROM offices;",
- "orderdetails" :   "SELECT * FROM orderdetails;",
- "orders"       :   "SELECT * FROM orders;",
- "payments"     :   "SELECT * FROM payments;",
- "productlines" :   "SELECT * FROM productlines;",
- "products"     :   "SELECT * FROM products;"
+### Built With
 
+* [Python 3.9.12](https://www.python.org/downloads/)
+* [google-cloud-storage 2.5.0](https://pypi.org/project/google-cloud-storage/)
 
- {
- "transactions"     :   "SELECT * FROM transactions;",
- "bookings"         :   "SELECT * FROM bookings;",
- "service_types"    :   "SELECT * FROM service_types;"
-}
+## ✨ Code-base structure
+
+### How To Run CMD:
+##### python run_extract_load.py

general_function.py

@@ -10,3 +10,13 @@ def read_json_file(path, name):
         print(e)
         print("[FUNCTION_ERROR]-read_json_file:" , name)
         return False
+
+
+def delete_csv_in_path(path):
+    for folder, subfolders, files in os.walk(path): 
+        for file in files: 
+            if file.endswith('.csv'): 
+                path = os.path.join(folder, file) 
+                print('deleted : ', path )
+                os.remove(path)
+                print("[Delete Temp Complete]",files)

requirements.txt

@@ -1,2 +1,2 @@
 mysql_connector_repackaged==0.3.1
-pandas==1.4.3
+google-cloud-storage==2.5.0

run_extract_load.py

@@ -30,24 +30,23 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH):
     #Connect to cloudstorage
     ex_storage_client = ex_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
 
+    #Get the filename and path in temp csv storage
     temp_file_list = []
     for (dirpath, dirnames, filenames) in walk(EXCHANGE_CSV_TEMP_STORAGE_PATH):
         temp_file_list.extend(filenames)
     
-    if temp_file_list == []:
-        print("No Exchange CSV Found")
-
+    #Loop all the file and upload to cloudstorage
     for file_name in temp_file_list:
         file_path = os.path.join(dirpath,file_name)
         ex_load_func.upload_to_bucket(ex_storage_client, file_name, file_path)
 
 
 def main():
-    #Extract Load Exchange Data
+    #Extract Load Exchange Data, and delete the temp csv
     EXCHANGE_CSV_TEMP_STORAGE_PATH = "exchange/extract/exchange_temp_extract_csv/"
     exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH)
     exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH)
-
+    general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH)
 
 
 if __name__ == "__main__":