From 945ea5ce4bd53ecddf0d5b83b1a81d68a5512e46 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Sun, 25 Sep 2022 19:25:45 +0800 Subject: [PATCH] Bug fix --- .gitignore | 6 +----- CLOUD_STORAGE_ADMIN_CREDENTIAL.json | 12 ++++++++++++ EXCHANGE_EXTRACT_SQL.json | 5 +++++ __pycache__/general_function.cpython-39.pyc | Bin 0 -> 923 bytes .../exchange_extract_function.cpython-39.pyc | Bin 0 -> 1698 bytes .../exchange_load_function.cpython-39.pyc | Bin 0 -> 985 bytes main.py | 2 ++ testing_sql.txt | 15 +++++++++++++++ 8 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 CLOUD_STORAGE_ADMIN_CREDENTIAL.json create mode 100644 EXCHANGE_EXTRACT_SQL.json create mode 100644 __pycache__/general_function.cpython-39.pyc create mode 100644 exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc create mode 100644 exchange/load/__pycache__/exchange_load_function.cpython-39.pyc create mode 100644 testing_sql.txt diff --git a/.gitignore b/.gitignore index 6f2c3fb..8a1b018 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1 @@ -*.json -*.csv -__pycache__ -testing_sql.txt -config/EXCHANGE_CONFIG.json +EXCHANGE_CONFIG.json diff --git a/CLOUD_STORAGE_ADMIN_CREDENTIAL.json b/CLOUD_STORAGE_ADMIN_CREDENTIAL.json new file mode 100644 index 0000000..909b7fd --- /dev/null +++ b/CLOUD_STORAGE_ADMIN_CREDENTIAL.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "data-management-359208", + "private_key_id": "06a6c2d224dff316c66498d62515b35ebdbe946f", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDbnWVbVtvG5Cbt\n8p3eriMFcQeFqFS1xp4srM09pFqmnvYMOJw8aJuK7RoRNlZQWKTgz8MabMGYaakL\nJJeiI6bUZO8EDAkui/TyiAseqJjOPBTgavHklyUKFSy/hdzLqRseEmi8SD1VU/Ek\nlzMARsnXw0Zrd+eptsR2MuKWQX4Qjj0FH1FDW/3GP2jgzS+HRktPzT3QIHlhyAOV\nxaCYoRY1IX/U8eNf0ZIu8SZzErLEjZRdhjMRuQVsvH4YZ11zbpI7UZZ/eLjT6zOB\nvbYJO6hXWFdTSej9FBg6e1ZJ82zVeMcFkaoNsfkD2CQqdv/3pXUbgtBLGCBuhNu3\nwQHyORIHAgMBAAECggEAaIm3oY7q9vXLgiCm/USu7vwqtHi4Of7ddC6dU+ZUMFQi\nkxavaCHzSGIsslzHIV/QvCKpoH58eOxyxxcYBtopo5iYHbkM9dcxNfGEOYfPlPwM\ng/bkRgecXfxOXKx/uYI5okrpCBbq+x8F/oDqigsoMUiG0Mk2wRZ61jjKmvN56q6o\nam59GBQoLOqh5uWWq/X0YqvsE8PaXqPvLwEZgeqTID6m2HAT6C4WSf+zuLe1vZ5O\nT19ue7ypWkgGD724ChQ/p/I9dy9P2nmtKDQ+4yGnHh8RU9WuXiSwRbDVJ6eV/C5J\nM27V1SoQmjAFYoQU+bq9tu0mHFPmiVELAVUPDW5oQQKBgQD4ZaBgLaRnGCBxfEBQ\n16B0Ks1NuxxSdyRk2aJ2OcDIXTLTcI1t5dJRgF836XqEoIHDrXaduVfhOqnbuJWv\nvcpbNhn3762hLCaih1bbz13GWIBqvVS8oJojJezC2r4tYgM3ydynx9C24YsBIhBl\nU2MBSufXUeDfI6ir+MLyNGuqjwKBgQDiVj111vqU7oABheeE1QewJMR4PdnKZWJ7\nGJHH1sBbaOeO0xYSFMg32fKvWLC2hEFJiNqWy90k7SmTcOPES2sV7nREk9MLO6Fy\nZyN4gw+5n3g+n66VAgTq2i99/vD95ubLH85GBfAt2KM07y3rgcNcEaP1QI9ue2lM\nHrVzNnQ9CQKBgQDOIvBTwKzljWUnKLjrHfafUQHtlvDrEsqWEvI68LSm0okiZQ5J\nfGbskf7zFIRDWjw2GlcMj0p5tEhP+j/mhzdOOHiWhEXwMgah7HTNl6o3tyxi6FpQ\n62re7lMsZYFbgjIvcwr2BeGUU1obB5zZqbjI0tPRobZfF2WbyaZmf9A1ywKBgQDg\nJ3q83sjSgJWjbIMKqZPwnak6UD8GVHxA3udZm9RrcyyI5YLhK1XTAnV3tQVl7Ptf\noTqix4nfTUW0sMPSHsMSOFNLq38Ci+7rhzu42UvUkRucIbbb+eD22ljYlokDXA9M\nMdauwKjKLtgLz6iRqbTZ1NqlRGgIig6RhYQ8czyRSQKBgDi+eLyxt+7zkyXMNhcO\nPEI4lOmjgSsHIKHByKlNx0EofmQYIu0zGwFWXd4MLApbfBy8zT2AqTwhv8XdhCn3\nsjwdNxpdHx1aJHFXkH1hndzzPz6Ud9oMMpTEcAgoYAoB6YV6OpLKQSGBUpbFzWdM\nYSuH1Vl4SdfL0p8/yS0UH7ed\n-----END PRIVATE KEY-----\n", + "client_email": "cloud-storage-admin-service-ac@data-management-359208.iam.gserviceaccount.com", + "client_id": "111008631797803896886", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/cloud-storage-admin-service-ac%40data-management-359208.iam.gserviceaccount.com" +} diff --git a/EXCHANGE_EXTRACT_SQL.json b/EXCHANGE_EXTRACT_SQL.json new file mode 100644 index 0000000..b79d775 --- /dev/null +++ b/EXCHANGE_EXTRACT_SQL.json @@ -0,0 +1,5 @@ +{ + "transactions" : "SELECT * FROM transactions;", + "bookings" : "SELECT * FROM bookings;", + "service_types" : "SELECT * FROM service_types;" +} \ No newline at end of file diff --git a/__pycache__/general_function.cpython-39.pyc b/__pycache__/general_function.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c6b4d8d153b6c1c404e93c387a408b83d71bd4ce GIT binary patch literal 923 zcmY*XO;6h}820NpA#~88X+mhH9@hh?+p$7uQVSB3RwO=z2t`pA8z|%p$1NjDcIoc> z3+<7=6cTn3Lj!1jyC!QjMHfSeIlQ2-BsrP5tXqel3xbhN3v7?BB zl9Vu|Q~3U-qKeL}lFp&BPVpv&b7}!e*4|-59Rl?{Ls??}2(yYkOn_7jNYPSu$)*5A zVK1g4kW+GoRVwzCaeDM*bw~#-?A00+t@FK;LGO6~VBqf_9v&QCY)TPu|0Yip|8f+G z?q1u`EX_nB%{w$R8O@?J;CgxYODHlm!hoI0QKH(mW?7)FHA{k6c*}pQXeS-3Y^*eh zMk2`)51HZc~0$vh3S)xlMhwO* z+%;^_KYwJQ>y-;dO_ zbg@I^BON7j$(W85aXJxgyUuvdWg2lIy+%I%yfAZZ8%nW~Q8f(c035U}fC{ literal 0 HcmV?d00001 diff --git a/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc b/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb015d9b1a22a6248f912f3a17395200ccf47449 GIT binary patch literal 1698 zcmZuxOKTiQ5bo}I?o)~_gB3^+IT$wJjgG+x5!lrdK_U^!Mr5>NGMes{MxMv2yVv{h zj4#U7ko<$)WBx#XN6t-dfk4i=gpfe0XIF}CJVRG?S3m0Os;@d|G^z-`Uw<~kiH*>2 zdhq5)VDJSr@i_>JD2|b*HHmRTJc5@t!!ye{%d@CK&F_(CQ;XW59g2@p%lQL#qk|S! zc9^AMAVaJTtIq(fQFNe*0f^bB5?$ddl7qiZ1d^1}D5bfHsP)<`aE?#VF*1=frr%*y z0G8Ru(#**X#z>mf*7^i}3kVDHicsfx)!Rnf&;wNNfif>MA(c52j4O8-2rDtnWV4P}(2${O-0l`X^5VHrdho|#PPxx(xx#B1Q`@9XeM{~3T2{b`WQF8hZm z>+vx%WF0*03v%lB#y8!p=APp{9!cy7S=pQHlkJ%)=4AKkMXA{YTP=a0>{FT8U zjM7j8Y7eKX@zz}&Kp`H2K$u|r_W#GWY}~{y{Ja5kp$^dPw+0;zzD(zbkmwv2rcUUb zT+$qp4S0O zBX7(<$lcOIeSU9&G-c8+fmHW@K-0f~R!yK)8)(&B(W;_Px+7NeRr5rXt-7>AIn+v; zfnC$=&ADqh<#ydRx^9;gtRKu;ryb4$$+mh~Oc_7hy33Aa$?z^kp?EQ?e)h8PUlwiX zzg0lGNOcB|p@|I;7ZA;}x-JolHqnpd+61~|u8XeZRf{On-u8?zi_7s+)L~jxcj8FM zFiVEJk6`>KG&F0KZ1!mc6bFL)GV=jLdneAqAQrpOx!qIxP?nnblFE`J@TAOa$WpFDs0_H2vP)>( zt1NXk7Rn0a47RwirnTm|$KQPAclSF_`2#R`P8l0|?$ydpIN}1TeZ5THpiHT+Z{R(A z*WyQFPhZNbz+)4$G)S0dKy+6JaVADFKwllNrB7>o4C(|BGaQ%qB zo6iUDO`S$Awm~3lk~*|1cF8(1F=%XqWgY$v7*%ndtb+}-QT_Lx!9RgLEth`;O1Tg^ z9>|QB;q%r?SM9qekL$;SYd)5bM7i}1c>0atCFk6oB%`C4?P`JweG3=Y_3L7|&Hn)6 CZ@t|B literal 0 HcmV?d00001 diff --git a/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc b/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61fb23dac41e6ee293762026d21d0f5de83a96b2 GIT binary patch literal 985 zcmaKr%}(1u5P5Cr2s021__E#jnQheyCeqNYppkwScVII z5IFYUr}4F?zCxu+oi!mKDs?T-&diSI$DhsT=i3O@`VT7p5QKg@=5J&AP^?FgBy4OH{mi4i*Sc`s{zW4)d-B@U5T!NKaip&y#`vKd8G&R z$c-aJC1_V<%Fvq(fT0fzmn<#)njw|6#8<#|O3}acrB_n~ZAQ`W2QOyh4aVYBS!Vdx z;`!lz=B8ch=_HGlRY%3!e`t^E5N0*IeG&i%?IC2iH7jor>p}2x8dk$Tj$)PaL>V8Z zG#-dIpJvEiWX?qv&Y?Zz?(0EaBsg|U!9sj w8=HCX_Pbut8EQR@)k^)v)*ojLr#^pBpe|KcX}(Xi9LH+i{c5@EzeAd>zjM+3`Tzg` literal 0 HcmV?d00001 diff --git a/main.py b/main.py index c8febc4..ff61edd 100644 --- a/main.py +++ b/main.py @@ -42,6 +42,8 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH): def main(): #Extract Load Exchange Data, and delete the temp csv EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/" + if not os.path.exists(EXCHANGE_CSV_TEMP_STORAGE_PATH): + os.makedirs(EXCHANGE_CSV_TEMP_STORAGE_PATH) exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH) exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH) general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH) diff --git a/testing_sql.txt b/testing_sql.txt new file mode 100644 index 0000000..5236ed8 --- /dev/null +++ b/testing_sql.txt @@ -0,0 +1,15 @@ + "customers" : "SELECT * FROM customers;", + "employees" : "SELECT * FROM employees;", + "offices" : "SELECT * FROM offices;", + "orderdetails" : "SELECT * FROM orderdetails;", + "orders" : "SELECT * FROM orders;", + "payments" : "SELECT * FROM payments;", + "productlines" : "SELECT * FROM productlines;", + "products" : "SELECT * FROM products;" + + + { + "transactions" : "SELECT * FROM transactions;", + "bookings" : "SELECT * FROM bookings;", + "service_types" : "SELECT * FROM service_types;" +} \ No newline at end of file