From 276e1b87fbb63b4a3987ec76910af802ace749f3 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Mon, 22 Aug 2022 19:22:11 +0800 Subject: [PATCH] Exchange Extract,Load, and delete temp --- .gitignore | 3 ++- README.md | 22 ++++++++-------------- general_function.py | 10 ++++++++++ requirements.txt | 2 +- run_extract_load.py | 9 ++++----- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index a9c44ae..9464862 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.csv -__pycache__ \ No newline at end of file +__pycache__ +testing_sql.txt \ No newline at end of file diff --git a/README.md b/README.md index 18ed5fa..417e718 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,11 @@ -# Guide +# CIEF Exchange Extract Load - "customers" : "SELECT * FROM customers;", - "employees" : "SELECT * FROM employees;", - "offices" : "SELECT * FROM offices;", - "orderdetails" : "SELECT * FROM orderdetails;", - "orders" : "SELECT * FROM orders;", - "payments" : "SELECT * FROM payments;", - "productlines" : "SELECT * FROM productlines;", - "products" : "SELECT * FROM products;" +### Built With +* [Python 3.9.12](https://www.python.org/downloads/) +* [google-cloud-storage 2.5.0](https://pypi.org/project/google-cloud-storage/) - { - "transactions" : "SELECT * FROM transactions;", - "bookings" : "SELECT * FROM bookings;", - "service_types" : "SELECT * FROM service_types;" -} \ No newline at end of file +## ✨ Code-base structure + +### How To Run CMD: +##### python run_extract_load.py diff --git a/general_function.py b/general_function.py index 51c5d49..3841c6b 100644 --- a/general_function.py +++ b/general_function.py @@ -10,3 +10,13 @@ def read_json_file(path, name): print(e) print("[FUNCTION_ERROR]-read_json_file:" , name) return False + + +def delete_csv_in_path(path): + for folder, subfolders, files in os.walk(path): + for file in files: + if file.endswith('.csv'): + path = os.path.join(folder, file) + print('deleted : ', path ) + os.remove(path) + print("[Delete Temp Complete]",files) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a858868..939e2f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ mysql_connector_repackaged==0.3.1 -pandas==1.4.3 +google-cloud-storage==2.5.0 diff --git a/run_extract_load.py b/run_extract_load.py index 98ac835..3e6a0e5 100644 --- a/run_extract_load.py +++ b/run_extract_load.py @@ -30,24 +30,23 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH): #Connect to cloudstorage ex_storage_client = ex_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE) + #Get the filename and path in temp csv storage temp_file_list = [] for (dirpath, dirnames, filenames) in walk(EXCHANGE_CSV_TEMP_STORAGE_PATH): temp_file_list.extend(filenames) - if temp_file_list == []: - print("No Exchange CSV Found") - + #Loop all the file and upload to cloudstorage for file_name in temp_file_list: file_path = os.path.join(dirpath,file_name) ex_load_func.upload_to_bucket(ex_storage_client, file_name, file_path) def main(): - #Extract Load Exchange Data + #Extract Load Exchange Data, and delete the temp csv EXCHANGE_CSV_TEMP_STORAGE_PATH = "exchange/extract/exchange_temp_extract_csv/" exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH) exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH) - + general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH) if __name__ == "__main__":