diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..16f2dc5 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.csv \ No newline at end of file diff --git a/python_el/extract/exchange_extract/exchange_extract_main.py b/python_el/extract/exchange_extract/exchange_extract_main.py index d694b74..75f8689 100644 --- a/python_el/extract/exchange_extract/exchange_extract_main.py +++ b/python_el/extract/exchange_extract/exchange_extract_main.py @@ -1,11 +1,16 @@ -from ntpath import join from operator import concat -import pandas as pd import mysql.connector import json import os import csv +#Get all the Compulsory File +CURRENT_DIRECTORY_PATH = "\\python_el\\extract\\exchange_extract" +MYSQL_CONFIG_FILE = "EXCHANGE_CONFIG.json" +SQL_FILE = "EXCHANGE_EXTRACT_SQL.json" +CSV_STORAGE_PATH = "exchange_temp_extract_csv/" + +#Function def change_directory_to(path): try: current_directory = os.getcwd() @@ -55,11 +60,7 @@ def write_to_csv(path,dbname,tablename,cursor,data): print("[FUNCTION_ERROR]-write_to_csv:" , filename) -#Get all the Compulsory File -CURRENT_DIRECTORY_PATH = "\\python_el\\extract\\exchange_extract" -MYSQL_CONFIG_FILE = "EXCHANGE_CONFIG.json" -SQL_FILE = "EXCHANGE_EXTRACT_SQL.json" -CSV_STORAGE_PATH = "exchange_temp_extract_csv/" +#_______________MAIN_________________ #Change directory & open all file change_directory_to(CURRENT_DIRECTORY_PATH) diff --git a/python_el/extract/exchange_load/data-management-359208-6c514887e773.json b/python_el/extract/exchange_load/data-management-359208-6c514887e773.json new file mode 100644 index 0000000..ee0d45c --- /dev/null +++ b/python_el/extract/exchange_load/data-management-359208-6c514887e773.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "data-management-359208", + "private_key_id": "6c514887e773d5febd945655b02bb56396623933", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDKAgKEBSL06qKE\n9BJsnUndQrpUqzEEzeMVbFlOOP8G/Af6uBhnNp0fCU7C+bRwPV4Eyuyg36c3udn9\nMCUQbIT9fwF+/05NlmduPJg0DqrVVtzShHUv8lQTHHhKmIM9cmDKnVZ6vzVg2+pT\nE2P6289PR3FKZKZrZnsDNEbwIVfrSRqCQtfH4Kayqb1Y6IsGhog6NjbqFVyNN0pH\nlgxuHex/pHK2/U79rEq1tVmAJ1E4WNvJdZjt4MiHdkinvwD9pwAboZgxYnBDPynm\n2ElrYSzpvT4nLf8c0TpT0/dnd5BF/Eh15wyEcowdzR6xnqnfqKHOJzB+JTr7AKt8\nagr5UlvhAgMBAAECggEAEP7+SzFLcaPULK+EZVMOhek5WCpXI3pXItRM50HwYxwN\nZ9DZbMWxjozv7YOo5NCk+m5AXoCyxwOCDcVhOPKIdfObop3Ebs66wRGkFK0vPmfi\niGvQmEohPMJmdJBEaoUXE7UNM6Km0RFvs7Gr9c1MsfTm2UWCowKqUuixFz8W8Jq9\npV3VImRXr2P/Ftc2wrnabRIICgGIlzyuS3AAC+OI8QESavOyEh7Z9wGckRLL4YNq\nmNFfH9/PGq8qD9g7DEz+SC4VraTKga62AeadvSeRiYN9FuO5e32H6rag9D2d+0HM\n516SZjYPS7mkubI+7mbhSBhJRDk4bs0q5TxcFqyQ4QKBgQDxCNFvcvrdD4Kf8yO3\nD2wTwk1ukF/+tc2Kj1K84Z6IKuMTQtH7x1HgwoPssfuRjJLDwBkYELyAVgMSHh39\n3aw3KrxUw05wSSr4M4v282ykQcfThQjGkRgYBpZuRcVCqarAbC6eicM6lGots2QR\nofmUxBIX1Hc7eK8TQwBlXNlVjQKBgQDWjN+yDDd+/qEaeuz1rJsBSS6mjLSYlSYU\nZctd2oPPdc+bQEIeqkkjkqTc8R0gUXV9s5CwuAymyf9jhnVUQU//k+5epiP2wyUz\nb6vLGOeAhwpfqLNFZWuI/M2RioC1YaUAC2qJDnEJNoRBV8nXEsszV9JdEqzyOtba\n4AdAoV0YpQKBgQCZB/k4oi6l9XgAp3UQf6klrmJNBTr9U14JT8++/hwR5fC/xNfe\n3ACfC8CIocPP+AkiYS9NeSrE7FcMxLRT/s6dQ/PIeSuu3LV8WfXON2TNsLn3EGqu\n72X1sxEFOCTymxg/DTBYFa0u3xW+qDurekQkcIvwN0PwLUIyn4J72IRf7QKBgQCl\nwWBxZg7aBk7g7mdzxk5ax/dKpRpBZ7lruNlNQSzkcthZ0WND3btzyC+mooEmHsju\nvHPkk8zybszoT1EGLw9nHRrj9OeEFXAANR48Ypk4KxxQmz0lOB3ET8thzedyOmYH\nispb6NRbkcrL0M8XYmWq3Qag8XS8D8k+gCYaQJB0IQKBgQCIiwdoeS5c9M2IATaJ\nF8fKDSKKo44Zql6TwLOGFmxu/ENrPFDyz2OpbqQDCG1kWnY8dhtdCVNsUxDaNp99\nAKMJN2/lQCdZ7mdlm776jIVJi2lppYv+SuxvcKBNS1LcFnKQzhKZtXKJbzFW94Kw\noB/54kQof7RvtQdFGhhE2CIj4Q==\n-----END PRIVATE KEY-----\n", + "client_email": "exchange-python-el-bigquery@data-management-359208.iam.gserviceaccount.com", + "client_id": "102572291647345024418", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/exchange-python-el-bigquery%40data-management-359208.iam.gserviceaccount.com" +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a858868 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +mysql_connector_repackaged==0.3.1 +pandas==1.4.3