diff --git a/.gitignore b/.gitignore index 6f2c3fb..8a1b018 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1 @@ -*.json -*.csv -__pycache__ -testing_sql.txt -config/EXCHANGE_CONFIG.json +EXCHANGE_CONFIG.json diff --git a/CLOUD_STORAGE_ADMIN_CREDENTIAL.json b/CLOUD_STORAGE_ADMIN_CREDENTIAL.json new file mode 100644 index 0000000..909b7fd --- /dev/null +++ b/CLOUD_STORAGE_ADMIN_CREDENTIAL.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "data-management-359208", + "private_key_id": "06a6c2d224dff316c66498d62515b35ebdbe946f", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDbnWVbVtvG5Cbt\n8p3eriMFcQeFqFS1xp4srM09pFqmnvYMOJw8aJuK7RoRNlZQWKTgz8MabMGYaakL\nJJeiI6bUZO8EDAkui/TyiAseqJjOPBTgavHklyUKFSy/hdzLqRseEmi8SD1VU/Ek\nlzMARsnXw0Zrd+eptsR2MuKWQX4Qjj0FH1FDW/3GP2jgzS+HRktPzT3QIHlhyAOV\nxaCYoRY1IX/U8eNf0ZIu8SZzErLEjZRdhjMRuQVsvH4YZ11zbpI7UZZ/eLjT6zOB\nvbYJO6hXWFdTSej9FBg6e1ZJ82zVeMcFkaoNsfkD2CQqdv/3pXUbgtBLGCBuhNu3\nwQHyORIHAgMBAAECggEAaIm3oY7q9vXLgiCm/USu7vwqtHi4Of7ddC6dU+ZUMFQi\nkxavaCHzSGIsslzHIV/QvCKpoH58eOxyxxcYBtopo5iYHbkM9dcxNfGEOYfPlPwM\ng/bkRgecXfxOXKx/uYI5okrpCBbq+x8F/oDqigsoMUiG0Mk2wRZ61jjKmvN56q6o\nam59GBQoLOqh5uWWq/X0YqvsE8PaXqPvLwEZgeqTID6m2HAT6C4WSf+zuLe1vZ5O\nT19ue7ypWkgGD724ChQ/p/I9dy9P2nmtKDQ+4yGnHh8RU9WuXiSwRbDVJ6eV/C5J\nM27V1SoQmjAFYoQU+bq9tu0mHFPmiVELAVUPDW5oQQKBgQD4ZaBgLaRnGCBxfEBQ\n16B0Ks1NuxxSdyRk2aJ2OcDIXTLTcI1t5dJRgF836XqEoIHDrXaduVfhOqnbuJWv\nvcpbNhn3762hLCaih1bbz13GWIBqvVS8oJojJezC2r4tYgM3ydynx9C24YsBIhBl\nU2MBSufXUeDfI6ir+MLyNGuqjwKBgQDiVj111vqU7oABheeE1QewJMR4PdnKZWJ7\nGJHH1sBbaOeO0xYSFMg32fKvWLC2hEFJiNqWy90k7SmTcOPES2sV7nREk9MLO6Fy\nZyN4gw+5n3g+n66VAgTq2i99/vD95ubLH85GBfAt2KM07y3rgcNcEaP1QI9ue2lM\nHrVzNnQ9CQKBgQDOIvBTwKzljWUnKLjrHfafUQHtlvDrEsqWEvI68LSm0okiZQ5J\nfGbskf7zFIRDWjw2GlcMj0p5tEhP+j/mhzdOOHiWhEXwMgah7HTNl6o3tyxi6FpQ\n62re7lMsZYFbgjIvcwr2BeGUU1obB5zZqbjI0tPRobZfF2WbyaZmf9A1ywKBgQDg\nJ3q83sjSgJWjbIMKqZPwnak6UD8GVHxA3udZm9RrcyyI5YLhK1XTAnV3tQVl7Ptf\noTqix4nfTUW0sMPSHsMSOFNLq38Ci+7rhzu42UvUkRucIbbb+eD22ljYlokDXA9M\nMdauwKjKLtgLz6iRqbTZ1NqlRGgIig6RhYQ8czyRSQKBgDi+eLyxt+7zkyXMNhcO\nPEI4lOmjgSsHIKHByKlNx0EofmQYIu0zGwFWXd4MLApbfBy8zT2AqTwhv8XdhCn3\nsjwdNxpdHx1aJHFXkH1hndzzPz6Ud9oMMpTEcAgoYAoB6YV6OpLKQSGBUpbFzWdM\nYSuH1Vl4SdfL0p8/yS0UH7ed\n-----END PRIVATE KEY-----\n", + "client_email": "cloud-storage-admin-service-ac@data-management-359208.iam.gserviceaccount.com", + "client_id": "111008631797803896886", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/cloud-storage-admin-service-ac%40data-management-359208.iam.gserviceaccount.com" +} diff --git a/EXCHANGE_EXTRACT_SQL.json b/EXCHANGE_EXTRACT_SQL.json new file mode 100644 index 0000000..b79d775 --- /dev/null +++ b/EXCHANGE_EXTRACT_SQL.json @@ -0,0 +1,5 @@ +{ + "transactions" : "SELECT * FROM transactions;", + "bookings" : "SELECT * FROM bookings;", + "service_types" : "SELECT * FROM service_types;" +} \ No newline at end of file diff --git a/__pycache__/general_function.cpython-39.pyc b/__pycache__/general_function.cpython-39.pyc new file mode 100644 index 0000000..c6b4d8d Binary files /dev/null and b/__pycache__/general_function.cpython-39.pyc differ diff --git a/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc b/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc new file mode 100644 index 0000000..fb015d9 Binary files /dev/null and b/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc differ diff --git a/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc b/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc new file mode 100644 index 0000000..61fb23d Binary files /dev/null and b/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc differ diff --git a/main.py b/main.py index c8febc4..ff61edd 100644 --- a/main.py +++ b/main.py @@ -42,6 +42,8 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH): def main(): #Extract Load Exchange Data, and delete the temp csv EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/" + if not os.path.exists(EXCHANGE_CSV_TEMP_STORAGE_PATH): + os.makedirs(EXCHANGE_CSV_TEMP_STORAGE_PATH) exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH) exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH) general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH) diff --git a/testing_sql.txt b/testing_sql.txt new file mode 100644 index 0000000..5236ed8 --- /dev/null +++ b/testing_sql.txt @@ -0,0 +1,15 @@ + "customers" : "SELECT * FROM customers;", + "employees" : "SELECT * FROM employees;", + "offices" : "SELECT * FROM offices;", + "orderdetails" : "SELECT * FROM orderdetails;", + "orders" : "SELECT * FROM orders;", + "payments" : "SELECT * FROM payments;", + "productlines" : "SELECT * FROM productlines;", + "products" : "SELECT * FROM products;" + + + { + "transactions" : "SELECT * FROM transactions;", + "bookings" : "SELECT * FROM bookings;", + "service_types" : "SELECT * FROM service_types;" +} \ No newline at end of file