From 47db302e191d99725403fd9af82b4fbe7084bc47 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Wed, 17 Aug 2022 19:46:57 +0800 Subject: [PATCH] Exchange Extract MYSQL Code --- meltano-el/.gitignore | 3 - meltano-el/README.md | 0 meltano-el/analyze/.gitkeep | 0 meltano-el/extract/.gitkeep | 0 meltano-el/load/.gitkeep | 0 meltano-el/meltano.yml | 7 -- meltano-el/notebook/.gitkeep | 0 meltano-el/orchestrate/.gitkeep | 0 meltano-el/output/.gitignore | 2 - meltano-el/requirements.txt | 0 meltano-el/transform/.gitkeep | 0 .../exchange_extract/EXCHANGE_CONFIG.json | 6 ++ .../EXCHANGE_EXTRACT_SQL.json | 10 +++ .../exchange_extract/exchange_extract_main.py | 90 +++++++++++++++++++ 14 files changed, 106 insertions(+), 12 deletions(-) delete mode 100644 meltano-el/.gitignore delete mode 100644 meltano-el/README.md delete mode 100644 meltano-el/analyze/.gitkeep delete mode 100644 meltano-el/extract/.gitkeep delete mode 100644 meltano-el/load/.gitkeep delete mode 100644 meltano-el/meltano.yml delete mode 100644 meltano-el/notebook/.gitkeep delete mode 100644 meltano-el/orchestrate/.gitkeep delete mode 100644 meltano-el/output/.gitignore delete mode 100644 meltano-el/requirements.txt delete mode 100644 meltano-el/transform/.gitkeep create mode 100644 python_el/extract/exchange_extract/EXCHANGE_CONFIG.json create mode 100644 python_el/extract/exchange_extract/EXCHANGE_EXTRACT_SQL.json create mode 100644 python_el/extract/exchange_extract/exchange_extract_main.py diff --git a/meltano-el/.gitignore b/meltano-el/.gitignore deleted file mode 100644 index 15e24c3..0000000 --- a/meltano-el/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/venv -/.meltano -.env diff --git a/meltano-el/README.md b/meltano-el/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/meltano-el/analyze/.gitkeep b/meltano-el/analyze/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/meltano-el/extract/.gitkeep b/meltano-el/extract/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/meltano-el/load/.gitkeep b/meltano-el/load/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/meltano-el/meltano.yml b/meltano-el/meltano.yml deleted file mode 100644 index 8ef3595..0000000 --- a/meltano-el/meltano.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 1 -default_environment: dev -environments: -- name: dev -- name: staging -- name: prod -project_id: 24ae7352-4d74-4ba6-988a-389fdd47eb74 diff --git a/meltano-el/notebook/.gitkeep b/meltano-el/notebook/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/meltano-el/orchestrate/.gitkeep b/meltano-el/orchestrate/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/meltano-el/output/.gitignore b/meltano-el/output/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/meltano-el/output/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/meltano-el/requirements.txt b/meltano-el/requirements.txt deleted file mode 100644 index e69de29..0000000 diff --git a/meltano-el/transform/.gitkeep b/meltano-el/transform/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/python_el/extract/exchange_extract/EXCHANGE_CONFIG.json b/python_el/extract/exchange_extract/EXCHANGE_CONFIG.json new file mode 100644 index 0000000..58f0034 --- /dev/null +++ b/python_el/extract/exchange_extract/EXCHANGE_CONFIG.json @@ -0,0 +1,6 @@ +{ + "host":"127.0.0.1", + "user":"admin", + "password":"admin", + "database":"classicmodels" +} \ No newline at end of file diff --git a/python_el/extract/exchange_extract/EXCHANGE_EXTRACT_SQL.json b/python_el/extract/exchange_extract/EXCHANGE_EXTRACT_SQL.json new file mode 100644 index 0000000..eef1ddc --- /dev/null +++ b/python_el/extract/exchange_extract/EXCHANGE_EXTRACT_SQL.json @@ -0,0 +1,10 @@ +{ + "customers" : "SELECT * FROM customers;", + "employees" : "SELECT * FROM employees;", + "offices" : "SELECT * FROM offices;", + "orderdetails" : "SELECT * FROM orderdetails;", + "orders" : "SELECT * FROM orders;", + "payments" : "SELECT * FROM payments;", + "productlines" : "SELECT * FROM productlines;", + "products" : "SELECT * FROM products;" +} \ No newline at end of file diff --git a/python_el/extract/exchange_extract/exchange_extract_main.py b/python_el/extract/exchange_extract/exchange_extract_main.py new file mode 100644 index 0000000..d694b74 --- /dev/null +++ b/python_el/extract/exchange_extract/exchange_extract_main.py @@ -0,0 +1,90 @@ +from ntpath import join +from operator import concat +import pandas as pd +import mysql.connector +import json +import os +import csv + +def change_directory_to(path): + try: + current_directory = os.getcwd() + target_dir = concat(current_directory,path) + return os.chdir(target_dir) + + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-change_directory_to") + + + +def read_json_file(name): + try: + with open(name) as json_file: + output = json.load(json_file) + return output + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-read_json_file:" , name) + + + +def connect_mysql(config): + mydb = mysql.connector.connect( + host = config["host"], + user = config["user"], + password = config["password"], + database = config["database"]) + return mydb + + + +def write_to_csv(path,dbname,tablename,cursor,data): + try: + filename = '/{}_{}.csv'.format(dbname, tablename) + + headers = [col[0] for col in cursor.description] # get headers + data.insert(0, tuple(headers)) + fp = open(concat(path,filename), 'w', newline = '') + myFile = csv.writer(fp) + myFile.writerows(data) + fp.close() + + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-write_to_csv:" , filename) + + +#Get all the Compulsory File +CURRENT_DIRECTORY_PATH = "\\python_el\\extract\\exchange_extract" +MYSQL_CONFIG_FILE = "EXCHANGE_CONFIG.json" +SQL_FILE = "EXCHANGE_EXTRACT_SQL.json" +CSV_STORAGE_PATH = "exchange_temp_extract_csv/" + +#Change directory & open all file +change_directory_to(CURRENT_DIRECTORY_PATH) +MYSQL_CONFIG = read_json_file(MYSQL_CONFIG_FILE) +SQL_DICT = read_json_file(SQL_FILE) + + +#Connect DB +mydb = connect_mysql(MYSQL_CONFIG) +mycursor = mydb.cursor() + + +for tablename,sql in SQL_DICT.items(): + + try: + mycursor.execute(sql) + myresult = mycursor.fetchall() + dbname = mydb.database + write_to_csv(CSV_STORAGE_PATH,dbname,tablename,mycursor,myresult) + except Exception as e: + print(e) + print("[ERROR]-SQL ERROR",tablename,sql) + + +mydb.close() + + +