From 0def9a4c546a36cc811066566c36b7bc7a1efde8 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Tue, 29 Nov 2022 20:23:27 +0800 Subject: [PATCH] ETLT and CRISP --- EXCHANGE_EXTRACT_SQL_OLD.json | 113 +++++++++--------- __pycache__/general_function.cpython-39.pyc | Bin 1105 -> 1405 bytes config/CRISP_CONFIG.json | 5 + config/EXCHANGE_EXTRACT_SQL.json | 34 +++--- config/SHIPPING_EXTRACT_SQL.json | 32 ++--- .../crisp_extract_function.cpython-39.pyc | Bin 0 -> 1997 bytes crisp/extract/crisp_extract_function.py | 61 ++++++++++ crisp/extract/crisp_temp_extract_csv/.gitkeep | 0 .../crisp_operators.csv | 20 ++++ .../exchange_extract_function.cpython-39.pyc | Bin 1940 -> 1785 bytes general_function.py | 13 +- main.py | 28 ++++- requirements.txt | 3 +- .../shipping_extract_function.cpython-39.pyc | Bin 0 -> 1785 bytes .../shipping_load_function.cpython-39.pyc | Bin 0 -> 985 bytes 15 files changed, 217 insertions(+), 92 deletions(-) create mode 100644 config/CRISP_CONFIG.json create mode 100644 crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc create mode 100644 crisp/extract/crisp_extract_function.py create mode 100644 crisp/extract/crisp_temp_extract_csv/.gitkeep create mode 100644 crisp/extract/crisp_temp_extract_csv/crisp_operators.csv create mode 100644 shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc create mode 100644 shipping/load/__pycache__/shipping_load_function.cpython-39.pyc diff --git a/EXCHANGE_EXTRACT_SQL_OLD.json b/EXCHANGE_EXTRACT_SQL_OLD.json index 759c8f2..b8ddf9c 100644 --- a/EXCHANGE_EXTRACT_SQL_OLD.json +++ b/EXCHANGE_EXTRACT_SQL_OLD.json @@ -1,77 +1,82 @@ { - "bookings" : "SELECT id, company_id, marking, service_id, bank_id, fix_amount, fix_currency_id, convertible_currency_id, conversion_currency_id, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM bookings;", - "companies" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, reference, debtor, type, business_type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM companies;", - "contacts" : "SELECT id, company_id, country_id, CONCAT(SUBSTR(reference,1,2),SUBSTR(reference,5,1),REPEAT('',LOCATE('@',reference)-3),SUBSTR(reference,LOCATE('@',reference)-1,3),REPEAT('',LENGTH(reference)-LOCATE('.',REVERSE(reference))-LOCATE('@',reference)-1),SUBSTR(reference,LENGTH(reference)-LOCATE('.',REVERSE(reference)))) AS reference, REPLACE(phone,RIGHT(phone,round(length(phone)/2,0)+1),REPEAT('*', round(length(phone)/2+0.5,0))) as phone, CONCAT(SUBSTR(email,1,2),SUBSTR(email,5,1),REPEAT('',LOCATE('@',email)-3),SUBSTR(email,LOCATE('@',email)-1,3),REPEAT('',LENGTH(email)-LOCATE('.',REVERSE(email))-LOCATE('@',email)-1),SUBSTR(email,LENGTH(email)-LOCATE('.',REVERSE(email)))) AS email, CONCAT(SUBSTR(wechat_id,1,2),SUBSTR(wechat_id,5,1),REPEAT('',LOCATE('@',wechat_id)-3),SUBSTR(wechat_id,LOCATE('@',wechat_id)-1,3),REPEAT('',LENGTH(wechat_id)-LOCATE('.',REVERSE(wechat_id))-LOCATE('@',wechat_id)-1),SUBSTR(wechat_id,LENGTH(wechat_id)-LOCATE('.',REVERSE(wechat_id)))) AS wechat_id, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM contacts;", - "countries" : "SELECT *, now() AS '_source_loaded_at' FROM countries;", - "currencies" : "SELECT *, now() AS '_source_loaded_at' FROM currencies;", - "currency_rates" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rates;", - "currency_rate_logs" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rate_logs;", - "employees" : "SELECT *, now() AS '_source_loaded_at' FROM employees;", - "groups" : "SELECT *, now() AS '_source_loaded_at' FROM groups;", - "group_transactions" : "SELECT *, now() AS '_source_loaded_at' FROM group_transactions;", - "segments" : "SELECT *, now() AS '_source_loaded_at' FROM segments;", - "segment_companies" : "SELECT *, now() AS '_source_loaded_at' FROM segment_companies;", - "service_types" : "SELECT *, now() AS '_source_loaded_at' FROM service_types;", - "transactions" : "SELECT *, now() AS '_source_loaded_at' FROM transactions;", - "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM `transaction_detail`;", - "users" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, MD5(email) AS email_hash_id, type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM users;", - "user_email_verifications" : "SELECT id, MD5(email) AS email_hash_id, is_complete, is_active, is_sent, created_at, updated_at, now() AS '_source_loaded_at' FROM user_email_verifications;", - "wallets" : "SELECT id, owner_type, owner_id, CONCAT(SUBSTR(code,1,2),SUBSTR(code,5,1),REPEAT('',LOCATE('@',code)-3),SUBSTR(code,LOCATE('@',code)-1,3),REPEAT('',LENGTH(code)-LOCATE('.',REVERSE(code))-LOCATE('@',code)-1),SUBSTR(code,LENGTH(code)-LOCATE('.',REVERSE(code)))) AS code, currency_id, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM wallets;" + "activity_log" : "SELECT * FROM activity_log;", + "addresses" : "SELECT * FROM addresses;", + "announcements" : "SELECT * FROM announcements;", + "announcement_segment" : "SELECT * FROM announcement_segment;", + "banks" : "SELECT * FROM banks;", + "bank_logs" : "SELECT * FROM bank_logs;", + "bookings" : "SELECT * FROM bookings;", + "companies" : "SELECT * FROM companies;", + "contacts" : "SELECT * FROM contacts;", + "countries" : "SELECT * FROM countries;", + "currencies" : "SELECT * FROM currencies;", + "currency_logs" : "SELECT * FROM currency_logs;", + "currency_rates" : "SELECT * FROM currency_rates;", + "currency_rate_logs" : "SELECT * FROM currency_rate_logs;", + "districts" : "SELECT * FROM districts;", + "documents" : "SELECT * FROM documents;", + "employees" : "SELECT * FROM employees;", + "failed_jobs" : "SELECT * FROM failed_jobs;", + "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '
'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;", + "groups" : "SELECT * FROM groups;", + "group_transactions" : "SELECT * FROM group_transactions;", + "jobs" : "SELECT * FROM jobs;", + "migrations" : "SELECT * FROM migrations;", + "model_has_roles" : "SELECT * FROM model_has_roles;", + "notifications" : "SELECT * FROM notifications;", + "password_resets" : "SELECT * FROM password_resets;", + "permissions" : "SELECT * FROM permissions;", + "roles" : "SELECT * FROM roles;", + "role_has_permissions" : "SELECT * FROM role_has_permissions;", + "segments" : "SELECT * FROM segments;", + "segment_companies" : "SELECT * FROM segment_companies;", + "segment_constants" : "SELECT * FROM segment_constants;", + "service_types" : "SELECT * FROM service_types;", + "states" : "SELECT * FROM states;", + "transactions" : "SELECT * FROM transactions;", + "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', '
'),'|', '|'),'\\\\','\\') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', '
'),'|', '|'),'\\\\','\\') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;", + "users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;", + "user_email_verifications" : "SELECT * FROM user_email_verifications;", + "wallets" : "SELECT * FROM wallets;" } + + { "activity_log" : "SELECT * FROM activity_log;", "addresses" : "SELECT * FROM addresses;", "announcements" : "SELECT * FROM announcements;", "announcement_segment" : "SELECT * FROM announcement_segment;", - // "banks" : "SELECT * FROM banks;", - // "bank_logs" : "SELECT * FROM bank_logs;", - // "bookings" : "SELECT * FROM bookings;", "companies" : "SELECT * FROM companies;", - --"company_connections" : "SELECT * FROM company_connections;", - --"company_employees" : "SELECT * FROM company_employees;", - --"company_modules" : "SELECT * FROM company_modules;", - --"connection_segments" : "SELECT * FROM connection_segments;", + "company_connections" : "SELECT * FROM company_connections;", + "company_employees" : "SELECT * FROM company_employees;", + "company_modules" : "SELECT * FROM company_modules;", + "connection_segments" : "SELECT * FROM connection_segments;", "contacts" : "SELECT * FROM contacts;", - --"containers" : "SELECT * FROM countries", - --"container_packing_lists" : "SELECT * FROM container_packing_lists", + "containers" : "SELECT * FROM containers", + "container_packing_lists" : "SELECT * FROM container_packing_lists", "countries" : "SELECT * FROM countries;", "currencies" : "SELECT * FROM currencies;", - // "currency_logs" : "SELECT * FROM currency_logs;", - // "currency_rates" : "SELECT * FROM currency_rates;", - // "currency_rate_logs" : "SELECT * FROM currency_rate_logs;", "districts" : "SELECT * FROM districts;", "documents" : "SELECT * FROM documents;", - // "employees" : "SELECT * FROM employees;", "failed_jobs" : "SELECT * FROM failed_jobs;", - "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', ''),'|', ''),'\\\\','') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;", - // "groups" : "SELECT * FROM groups;", - // "group_transactions" : "SELECT * FROM group_transactions;", - "jobs" : "SELECT * FROM jobs;", + "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '
'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;", "migrations" : "SELECT * FROM migrations;", - // "model_has_roles" : "SELECT * FROM model_has_roles;", - // "notifications" : "SELECT * FROM notifications;", - --"orders" : "SELECT * FROM orders", - --"order_roles" : "SELECT * FROM order_roles", - --"packages" : "SELECT * FROM packages", - --"packing_lists" : "SELECT * FROM packing_lists", + "orders" : "SELECT * FROM orders", + "order_roles" : "SELECT * FROM order_roles", + "packages" : "SELECT * FROM packages", + "packing_lists" : "SELECT * FROM packing_lists", "password_resets" : "SELECT * FROM password_resets;", - // "permissions" : "SELECT * FROM permissions;", - --"remarks" : "SELECT * FROM remarks", - // "roles" : "SELECT * FROM roles;", - // "role_has_permissions" : "SELECT * FROM role_has_permissions;", - --"schedules" : "SELECT * FROM schedules", + "remarks" : "SELECT id, owner_type, owner_id, commenter_id, REPLACE(REPLACE(REPLACE(content, '\n', '
'),'|', '|'),'\\\\','\\') AS 'content', deleted_at, created_at, updated_at FROM remarks", + "schedules" : "SELECT * FROM schedules", "segments" : "SELECT * FROM segments;", - // "segment_companies" : "SELECT * FROM segment_companies;", "segment_constants" : "SELECT * FROM segment_constants;", - // "service_types" : "SELECT * FROM service_types;", "states" : "SELECT * FROM states;", - --"steps" : "SELECT * FROM steps", + "steps" : "SELECT * FROM steps", "transactions" : "SELECT * FROM transactions;", - "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;", - --"transport_arrangements" : "SELECT * FROM transport_arrangements", + "transaction_details" : "SELECT * FROM `transaction_details`;", + "transport_arrangements" : "SELECT * FROM transport_arrangements", "users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;", - "user_email_verifications" : "SELECT * FROM user_email_verifications;", - // "wallets" : "SELECT * FROM wallets;" -} \ No newline at end of file + "user_email_verifications" : "SELECT * FROM user_email_verifications;" +} diff --git a/__pycache__/general_function.cpython-39.pyc b/__pycache__/general_function.cpython-39.pyc index 710ebe6eb95ad3992472412d5d1e10f475f34807..8e8548e42e5d03b2d6b9ecc0e39db594a1a98468 100644 GIT binary patch delta 453 zcmY*V-AV#M6h3EmUB^|oAEb*?>&8Gr@Es&6Cf#dik=zJPeCyNKuy`2&e_DwfN2AL1w zS`;ITIZkns%#sphGbad}P)xpFJ|bFgUsym~o8^u7$tF7nruM}Qsx9UY>eC_&W_@dg zcW}fIt%mS~BY4I!j<5wyFd-DTRfMrXi1MK^68DG@8ibl#a0bM+J3*qk$jVUWjOQ_G zlc;cBJ*`zv4(qk{exp%uT$HW@f6yNI>c;C|McsDjx2abxnsYHE*I6zqi$_v%hN7YL zD_lW6mDFPDAJUx7Gs}&CuG>5W1P<2$1=?t#gOZoZu~M{u9p};?c9j?UN(olg3WIA6 iy#aZF%4$XTs|^kM`&4e0Gk4duC^4n&giOkWjQs$N01&Db4gdfE diff --git a/config/CRISP_CONFIG.json b/config/CRISP_CONFIG.json new file mode 100644 index 0000000..601b1e2 --- /dev/null +++ b/config/CRISP_CONFIG.json @@ -0,0 +1,5 @@ +{ + "identifier" : "0f8edfb8-1271-4e8c-bdfc-e33de84a0efe", + "key" : "408094e63ab670c968b3f9a119387263d4e23721817513b47bd94700bf4349a9", + "website_id" : "665dcd41-1edf-4451-8cb9-f1cf9ed35e15" +} \ No newline at end of file diff --git a/config/EXCHANGE_EXTRACT_SQL.json b/config/EXCHANGE_EXTRACT_SQL.json index f39a497..86ae5a9 100644 --- a/config/EXCHANGE_EXTRACT_SQL.json +++ b/config/EXCHANGE_EXTRACT_SQL.json @@ -1,43 +1,43 @@ { - "activity_log" : "SELECT * FROM activity_log;", - "addresses" : "SELECT * FROM addresses;", + "activity_log" : "SELECT ID, LOG_NAME, DESCRIPTION, SUBJECT_ID, SUBJECT_TYPE, CAUSER_ID, CAUSER_TYPE, '**MASKING_PROPERTIES**' AS PROPERTIES, CREATED_AT, UPDATED_AT FROM activity_log;", + "addresses" : "SELECT ID, COMPANY_ID, COUNTRY_ID, STATE_ID, DISTRICT_ID, POSTCODE, '**MASKING_STREET_ONE**'AS STREET_ONE, '**MASKING_STREET_TWO**'AS STREET_TWO, BILLING, DELETED_AT, CREATED_AT, UPDATED_AT FROM addresses;", "announcements" : "SELECT * FROM announcements;", "announcement_segment" : "SELECT * FROM announcement_segment;", - "banks" : "SELECT * FROM banks;", - "bank_logs" : "SELECT * FROM bank_logs;", + "banks" : "SELECT ID, COMPANY_ID, '**MASKING_REFERENCE**' AS REFERENCE, BANK_NAME, MD5(CONCAT(HOLDER_NAME,LENGTH(HOLDER_NAME)*2022*2)) AS HOLDER_NAME, MD5(CONCAT(ACCOUNT_NO,LENGTH(ACCOUNT_NO)*2022*2)) AS ACCOUNT_NO, BANK_BRANCH, MD5(CONCAT(SWIFT,LENGTH(SWIFT)*2022*2)) AS SWIFT, MD5(CONCAT(SNAP,LENGTH(SNAP)*2022*2)) AS SNAP, TYPE, DEFAULT, STATUS, COUNTRY_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM banks;", + "bank_logs" : "SELECT ID, BANK_ID, COMPANY_ID, '**MASKING_REFERENCE**' AS REFERENCE, BANK_NAME, MD5(CONCAT(HOLDER_NAME,LENGTH(HOLDER_NAME)*2022*2)) AS HOLDER_NAME, MD5(CONCAT(ACCOUNT_NO,LENGTH(ACCOUNT_NO)*2022*2)) AS ACCOUNT_NO, BANK_BRANCH, MD5(CONCAT(SWIFT,LENGTH(SWIFT)*2022*2)) AS SWIFT, MD5(CONCAT(SNAP,LENGTH(SNAP)*2022*2)) AS SNAP, TYPE, DEFAULT, STATUS, COUNTRY_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM bank_logs;", "bookings" : "SELECT * FROM bookings;", - "companies" : "SELECT * FROM companies;", - "contacts" : "SELECT * FROM contacts;", + "companies" : "SELECT ID, '**MASKING_NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, DEBTOR, TYPE, BUSINESS_TYPE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM companies;", + "contacts" : "SELECT ID, COMPANY_ID, COUNTRY_ID, '**MASKING_REFERENCE**' AS REFERENCE, MD5(CONCAT(PHONE,LENGTH(PHONE)*2022*2)) AS PHONE, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, MD5(CONCAT(WECHAT_ID,LENGTH(WECHAT_ID)*2022*2)) AS WECHAT_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM contacts;", "countries" : "SELECT * FROM countries;", "currencies" : "SELECT * FROM currencies;", "currency_logs" : "SELECT * FROM currency_logs;", "currency_rates" : "SELECT * FROM currency_rates;", "currency_rate_logs" : "SELECT * FROM currency_rate_logs;", "districts" : "SELECT * FROM districts;", - "documents" : "SELECT * FROM documents;", + "documents" : "SELECT ID, OWNER_TYPE, OWNER_ID, DOCUMENT_TYPE, '**MASKING_REFERENCE**' AS REFERENCE, STATUS, ISSUED_DATE, EXPIRED_DATE, APPROVER, APPROVAL_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM documents;", "employees" : "SELECT * FROM employees;", - "failed_jobs" : "SELECT * FROM failed_jobs;", - "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '
'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;", - "groups" : "SELECT * FROM groups;", + "failed_jobs" : "SELECT ID, CONNECTION, QUEUE, '**MASKING_PAYLOAD' AS PAYLOAD, '**MASKING_EXCEPTION**' AS EXCEPTION, FAILED_AT FROM failed_jobs;", + "files" : "SELECT id, document_id, '**MASKING_FILE**', file_type, deleted_at, created_at, updated_at FROM files;", + "groups" : "SELECT ID, '**MASKING_REFERENCE**' AS REFERENCE, ISSUER, RECEIVER, AMOUNT, ORIGINAL_AMOUNT, CURRENCY_ID, ORIGINAL_CURRENCY_ID, CURRENCY_RATE, TAX, SERVICE_CHARGE, STATUS, CREATED_AT, UPDATED_AT FROM groups;", "group_transactions" : "SELECT * FROM group_transactions;", - "jobs" : "SELECT * FROM jobs;", + "jobs" : "SELECT ID, QUEUE, '**MASKING_PAYLOAD**' AS PAYLOAD, ATTEMPTS, RESERVED_AT, AVAILABLE_AT, CREATED_AT FROM jobs;", "migrations" : "SELECT * FROM migrations;", "model_has_roles" : "SELECT * FROM model_has_roles;", "notifications" : "SELECT * FROM notifications;", - "password_resets" : "SELECT * FROM password_resets;", + "password_resets" : "SELECT ID, USER_ID, '**MASKING_TOKEN**' AS TOKEN, IS_EXPIRED, IS_COMPLETE, CREATED_AT, UPDATED_AT FROM password_resets;", "permissions" : "SELECT * FROM permissions;", "roles" : "SELECT * FROM roles;", "role_has_permissions" : "SELECT * FROM role_has_permissions;", "segments" : "SELECT * FROM segments;", "segment_companies" : "SELECT * FROM segment_companies;", - "segment_constants" : "SELECT * FROM segment_constants;", + "segment_constants" : "SELECT ID, SEGMENT_ID, NAME, REFERENCE, '**MASKING_DETAIL**' AS DETAIL, DELETED_AT, CREATED_AT, UPDATED_AT FROM segment_constants;", "service_types" : "SELECT * FROM service_types;", "states" : "SELECT * FROM states;", - "transactions" : "SELECT * FROM transactions;", + "transactions" : "SELECT ID, OWNER_TYPE, OWNER_ID, TYPE, ISSUER, RECEIVER, RECIPIENT_BANK_ACCOUNT_ID, PAYMENT_METHOD, '**PAYMENT_REFERENCE**' AS PAYMENT_REFERENCE, MD5(CONCAT(BILL_NO,LENGTH(BILL_NO)*2022*2)) AS BILL_NO, AMOUNT, ORIGINAL_AMOUNT, CURRENCY_ID, ORIGINAL_CURRENCY_ID, CURRENCY_RATE, TAX, SERVICE_CHARGE, EXPIRES_ON, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM transactions;", "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', '
'),'|', '|'),'\\\\','\\') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', '
'),'|', '|'),'\\\\','\\') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;", - "users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;", - "user_email_verifications" : "SELECT * FROM user_email_verifications;", - "wallets" : "SELECT * FROM wallets;" + "users" : "SELECT id, '**MASKING_NAME**' AS name, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;", + "user_email_verifications" : "SELECT ID, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, '**MASKING_TOKEN**' AS TOKEN, IS_COMPLETE, IS_ACTIVE, IS_SENT, CREATED_AT, UPDATED_AT FROM user_email_verifications;", + "wallets" : "SELECT ID, OWNER_TYPE, OWNER_ID, MD5(CONCAT(CODE,LENGTH(CODE)*2022*2)) AS CODE, CURRENCY_ID, AMOUNT, DELETED_AT, CREATED_AT, UPDATED_AT FROM wallets;" } diff --git a/config/SHIPPING_EXTRACT_SQL.json b/config/SHIPPING_EXTRACT_SQL.json index d49fbd9..a9deba7 100644 --- a/config/SHIPPING_EXTRACT_SQL.json +++ b/config/SHIPPING_EXTRACT_SQL.json @@ -1,37 +1,37 @@ { - "activity_log" : "SELECT * FROM activity_log;", - "addresses" : "SELECT * FROM addresses;", + "activity_log" : "SELECT ID, LOG_NAME, DESCRIPTION, SUBJECT_TYPE, SUBJECT_ID, CAUSER_TYPE, CAUSER_ID, '**MASKING_PROPERTIES**' AS PROPERTIES, CREATED_AT, UPDATED_AT FROM activity_log;", + "addresses" : "SELECT ID, OWNER_TYPE, OWNER_ID, '**MASKING_REFERENCE**' AS REFERENCE, '**MASKING_STREET_ONE**' AS STREET_ONE, '**MASKING_STREET_TWO**'AS STREET_TWO, DISTRICT_ID, STATE_ID, COUNTRY_ID, POSTCODE, TYPE, DEFAULT, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM addresses;", "announcements" : "SELECT * FROM announcements;", "announcement_segment" : "SELECT * FROM announcement_segment;", - "companies" : "SELECT * FROM companies;", - "company_connections" : "SELECT * FROM company_connections;", + "companies" : "SELECT ID, '**MASKING_NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, DEBTOR, TYPE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM companies;", + "company_connections" : "SELECT ID, INVITER_ID, INVITEE_ID, '**MASKING INVITER_REFERENCE**' AS INVITER_REFERENCE, '**MASKING INVITEE_REFERENCE**' AS INVITEE_REFERENCE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM company_connections;", "company_employees" : "SELECT * FROM company_employees;", - "company_modules" : "SELECT * FROM company_modules;", + "company_modules" : "SELECT ID, COMPANY_ID, '**MASKING NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, TYPE, STATUS, MD5(CONCAT(UNITY_HASH_ID,LENGTH(UNITY_HASH_ID)*2022*2)) AS UNITY_HASH_ID, MD5(CONCAT(UNITY_SIGNATURE,LENGTH(UNITY_SIGNATURE)*2022*2)) AS UNITY_SIGNATURE, DELETED_AT, CREATED_AT, UPDATED_AT FROM company_modules;", "connection_segments" : "SELECT * FROM connection_segments;", - "contacts" : "SELECT * FROM contacts;", + "contacts" : "SELECT ID, OWNER_TYPE, OWNER_ID, '**MASKING_REFERENCE**' AS REFERENCE, MD5(CONCAT(PHONE,LENGTH(PHONE)*2022*2)) AS PHONE, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, MD5(CONCAT(WECHAT_ID,LENGTH(WECHAT_ID)*2022*2)) AS WECHAT_ID, DEFAULT, DELETED_AT, CREATED_AT, UPDATED_AT FROM contacts;", "containers" : "SELECT * FROM containers", "container_packing_lists" : "SELECT * FROM container_packing_lists", "countries" : "SELECT * FROM countries;", "currencies" : "SELECT * FROM currencies;", "districts" : "SELECT * FROM districts;", - "documents" : "SELECT * FROM documents;", - "failed_jobs" : "SELECT * FROM failed_jobs;", - "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '
'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;", + "documents" : "SELECT ID, OWNER_TYPE, OWNER_ID, DOCUMENT_TYPE, '**MASKING_REFERENCE**' AS REFERENCE, STATUS, ISSUED_DATE, EXPIRED_DATE, APPROVER, APPROVAL_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM documents;", + "failed_jobs" : "SELECT ID, UUID, CONNECTION, QUEUE, '**MASKING_PAYLOAD' AS PAYLOAD, '**MASKING_EXCEPTION**' AS EXCEPTION, FAILED_AT FROM failed_jobs;", + "files" : "SELECT id, document_id, '**MASKING FILE**' AS FILE, file_type, deleted_at, created_at, updated_at FROM files;", "migrations" : "SELECT * FROM migrations;", "orders" : "SELECT * FROM orders", - "order_roles" : "SELECT * FROM order_roles", + "order_roles" : "SELECT ID, ORDER_ID, COMPANY_MODULE_ID, ROLE_ID, MD5(CONCAT(ENTITY_HASH_ID,LENGTH(ENTITY_HASH_ID)*2022*2)) AS ENTITY_HASH_ID. MD5(CONCAT(ENTITY_SIGNATURE,LENGTH(ENTITY_SIGNATURE)*2022*2)) AS ENTITY_SIGNATURE, DELETED_AT, CREATED_AT, UPDATED_AT FROM order_roles", "packages" : "SELECT * FROM packages", "packing_lists" : "SELECT * FROM packing_lists", - "password_resets" : "SELECT * FROM password_resets;", - "remarks" : "SELECT id, owner_type, owner_id, commenter_id, REPLACE(REPLACE(REPLACE(content, '\n', '
'),'|', '|'),'\\\\','\\') AS 'content', deleted_at, created_at, updated_at FROM remarks", + "password_resets" : "SELECT ID, USER_ID, '**MASKING_TOKEN**' AS TOKEN, IS_EXPIRED, IS_COMPLETE, CREATED_AT, UPDATED_AT FROM password_resets;", + "remarks" : "SELECT id, owner_type, owner_id, commenter_id, '**MASKING CONTENT**' AS CONTENT, deleted_at, created_at, updated_at FROM remarks", "schedules" : "SELECT * FROM schedules", "segments" : "SELECT * FROM segments;", "segment_constants" : "SELECT * FROM segment_constants;", "states" : "SELECT * FROM states;", - "steps" : "SELECT * FROM steps", + "steps" : "SELECT ID, OWNER_TYPE, OWNER_ID, APPOINTEE_ID, MD5(CONCAT(OBLIGATION_HASH_ID,LENGTH(OBLIGATION_HASH_ID)*2022*2)) AS OBLIGATION_HASH_ID, REFERENCE, PRIMARY, SEQUENCE, STATUS, COMPLETE_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM steps", "transactions" : "SELECT * FROM transactions;", "transaction_details" : "SELECT * FROM `transaction_details`;", - "transport_arrangements" : "SELECT * FROM transport_arrangements", - "users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;", - "user_email_verifications" : "SELECT * FROM user_email_verifications;" + "transport_arrangements" : "SELECT ID, OWNER_TYPE, OWNER_ID, TYPE, COURIER, LEFT(TRACKING_NUMBER,6) AS TRACKING_NUMBER, DISPATCH_DATE, DROP_DATE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM transport_arrangements", + "users" : "SELECT id, '**MASKING NAME**' AS NAME, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;", + "user_email_verifications" : "SELECT ID, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, '**MASKING_TOKEN**' AS TOKEN, IS_COMPLETE, IS_ACTIVE, IS_SENT, CREATED_AT, UPDATED_AT FROM user_email_verifications;" } diff --git a/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc b/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e86a95cc2c0eaea71ef642d3cdfa66714fc8500 GIT binary patch literal 1997 zcmZvdOK;pZ5P(VQMeW0n#BiEEQX{=|i#o`mNR1*0irO)dLv6rG5LDSzRYdNtY^|hF z+{9jJ4HW4$K!1XL%&+O4J@%AyfgXJ7jCQkj9x`?~Ga`p0&c|`oZZ`-#fBpS?{I5^Q zKWJQh7#KgouiOA3h@dGM(tuLbS;_)dk5*s{OW1#qz!8pcVRVHjIE4|g^n+}qs=@9ypG?mbzL^DL9G4rBa)?T(}HWCC7F9cow>w4#w7V30V{ z()f1|V_E1V&x}*3B-0(sxDcY3^kGaBm}sbcO%#6k*}>)y-G`-AW%no=jtAY{Om3;< zx$JJq@<8WBck|xe?Qkp7(a%v9_2m$jcLkX1!*jb%tSb+-ielZZt#D?Ay-^lpGB=7N z(|+em96a<(AP8mDr470Qa)EOAQ!VhuHoluZ9L9O2j=>oJRI$psLMC=a_sI#Z$SIvT znw?mc1Dad8`*vjuDj4E{XuNWvI*#xv>m^j|5y@^r1>K`VsGwI@(5bw-f{x%9D(D@1 zvkHPXub{tkMyoS_qm?kmt<^}L(d*l<9b=WUFkEJFE|LsReIO$N<@KM9axLRQ zq>PhfLLQ!7e!P{x$kIF#Yn%D7NTrrf#@9appbwSK3#lTVt1^ts=N->@FXYoQ(K1*~ zlTwE>>wKD7>dOq4r1J`xmHjf$!Yo(AC{4yP)Ojd+#)SbwDi;n_VNE*RG~R|wgu;_I zb`j};;k_i4Su~VpF(2t-q{B1^=AgR4<@5aNd~mB^`8}uTDY)oU@@o846N4{*@O?|* zE>}Px8l1KOj#WTo2FDus-+^NR9CJ9ng9|?X?=2h}&^RRk#&mQAoj_lh@X8bPfT;n% zgUFDIpR$Rs?GvZ+0iT8@`)<{!oIV4DtXFIT{_D=yggWWqsn_Ps6gUciHYeKwM9j}G)s%F&?&TTS_>(1l4&oYS8s9Z!;c&{wj$=PoLx?K38A({Yh z3(*Cjo1*=`S97s22YLg5Zv9D*ZUEBl8fjj&YoxhYynuB3xGn6uDS9+9;nFE|*B8;8Np6)QipuJdw{KXmlR^johMHE;!wG3j+wJh}wsoOur64sVVZ4Us2l+-)m7Z)QxJ7y0|M1o=s|*mMTBX# z0Q;12AXZv!of#Mp9FF^#0Z!@a56}@u@hJxD?_-UROVK0sPrddEg z471VjDfHmwcUIQ)ds+?)SW_n#EM3Sc#P=V)3Kz1kj|DR=Z-JOz1niAAwHb%2(4dz< zcUglqz>7AgD{P6a;#FZwU_l>Uhm{>3_|u*o6-oWSqdq|67EvamQhg0`>V~V#9p!g!On{s)kl2I>F+ literal 0 HcmV?d00001 diff --git a/crisp/extract/crisp_extract_function.py b/crisp/extract/crisp_extract_function.py new file mode 100644 index 0000000..735e888 --- /dev/null +++ b/crisp/extract/crisp_extract_function.py @@ -0,0 +1,61 @@ +from crisp_api import Crisp +import general_function +import pandas as pd + +def connect_crisp(config): + try: + client = Crisp() + client.set_tier("plugin") + client.authenticate(config["identifier"], config["key"]) + return client + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-connect_crisp") + return False + +def get_operators_csv(website_id, client, path, filename): + try: + output_location = path + filename + operators_list_json = client.website.list_website_operators(website_id) + operators_list_df = general_function.json_normalize_to_df(operators_list_json) + operators_list_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + print("[Download Complete]",filename) + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-get_operators_csv") + return False + +def get_peoples_csv(website_id, client, path, filename): + try: + output_location = path + filename + peoples_list_df = pd.DataFrame() + peoples_data_df = pd.DataFrame() + loop_flag = True + page_number = 1 + + while loop_flag == True: + peoples_list_json = client.website.list_people_profiles(website_id, page_number) + temp_df = general_function.json_normalize_to_df(peoples_list_json) + peoples_list_df = pd.concat([temp_df,peoples_list_df], ignore_index=True) + if not peoples_list_json: + loop_flag = False + else: + page_number += 1 + + for people_id in peoples_list_df.people_id: + print(people_id) + people_data_json = client.website.get_people_data(website_id, people_id) + temp_df = general_function.json_normalize_to_df(people_data_json) + temp_df['people_id'] = people_id + peoples_data_df = pd.concat([temp_df,peoples_data_df], ignore_index=True) + + peoples_list_df = peoples_list_df.merge(peoples_data_df, on='people_id', how='left') + + peoples_list_df.to_csv(output_location, sep=',', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + peoples_data_df.to_csv('test.csv', sep=',', encoding='utf-8-sig', header='true', quotechar= '"', index=False) + print("[Download Complete]",filename) + + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-get_peoples_csv") + return False \ No newline at end of file diff --git a/crisp/extract/crisp_temp_extract_csv/.gitkeep b/crisp/extract/crisp_temp_extract_csv/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/crisp/extract/crisp_temp_extract_csv/crisp_operators.csv b/crisp/extract/crisp_temp_extract_csv/crisp_operators.csv new file mode 100644 index 0000000..ccf4889 --- /dev/null +++ b/crisp/extract/crisp_temp_extract_csv/crisp_operators.csv @@ -0,0 +1,20 @@ +type|details.user_id|details.email|details.avatar|details.first_name|details.last_name|details.role|details.title|details.availability|details.has_token +operator|15f0ec35-8a0c-457f-83ad-68e04f3b6624|taypeisongtps@gmail.com|https://storage.crisp.chat/users/avatar/operator/5e71c78373d9ac00/potrait-photo_gsa6gd.jpg|Pei Song|Tay|member|Customer Service|away|False +operator|293073d9-e65f-40e2-a1ca-54fb9a2e4ff7|ykheng2bb@gmail.com|https://storage.crisp.chat/users/avatar/operator/4a7ce09c05d7b400/whatsapp-image-2022-04-13-at-2_19jwcch.jpeg|YAP|KAH HENG|owner|Customer Service|online|False +operator|294351f3-d501-41dd-a095-2c512dd2ff4f|ciefworldwide.acc1@gmail.com||Haizat|Zaidi|owner||away|False +operator|2a0b5503-97ff-4875-918d-be8185bc58a1|h2009413007@hcu.edu.my|https://storage.crisp.chat/users/avatar/operator/6bba6be706925c00/screenshot-2022-09-30-150004_mst2g.png|Tok|soon wee|member|Consultant|online|False +operator|413c155c-3f3b-4859-bc18-dddb17dfd3ac|diliniwr10@gmail.com|https://storage.crisp.chat/users/avatar/operator/ed6e6534f8f01000/dili_wntzti.png|Dilini|WR|member||away|False +operator|6f231d14-b3c0-47b5-bd02-617579cd2bcb|kangjun8331@gmail.com|https://storage.crisp.chat/users/avatar/operator/9ccd7bf346431800/img1413_d7cf1m.jpg|Kang|Jun|member|Consultant|away|False +operator|80708bf4-9c08-4b8d-a37b-3a2f3c9570f1|jhunyanwong@gmail.com|https://storage.crisp.chat/users/avatar/operator/b20140b4cfccf800/214025936361797548903332433376_1r1o1en.jpg|JY|Wong|owner||away|False +operator|83fb3451-f17a-4876-b317-244c30b27c8b|shafiqa_sukeri@cief-malaysia.com|https://storage.crisp.chat/users/avatar/operator/6ba11d171dddb800/photo2022-07-0811-15-15_9ulegi.jpg|FAIN|BACKEND|owner|Customer Service|online|False +operator|99a1c001-60ac-4f71-91e1-33e8c2205525|enxin1227@gmail.com|https://storage.crisp.chat/users/upload/operator/bb7c7077d72bc800/21064570005_zf9jf6.jpg|Tiffany|Ooi|member||online|False +operator|9df5d57c-8eb6-4e43-a8c5-cc6282564de9|ciefworldwide@gmail.com||PM|Wong|owner|CEO|away|False +operator|a8079229-94c0-4a33-a027-bcbaed17a1c0|uldvstar@gmail.com|https://storage.crisp.chat/users/avatar/operator/8f0372fcb5eb0000/download-1_17y48zb.png|Omair|Saleh|owner||away|False +operator|d8e200bc-709b-4b80-a706-bab8ec612e0b|infociefworldwide@gmail.com||CIEF|Marketing|owner|Marketing|away|False +operator|d9ef0024-60c9-4137-919d-1d25bc2fab0e|operationteam@cief-malaysia.com||Operation|Team|member||away|False +operator|db57d7a6-9421-43b9-9614-0867db22739a|ziwen38@gmail.com|https://storage.crisp.chat/users/avatar/operator/eed9c8d96123e00/resume_1x03t8n.jpeg|Zi Wen|Lee|member||online|False +operator|dd8d2dcb-07a4-4343-b5bd-93038d08bd54|lingzunsien1211@gmail.com|https://storage.crisp.chat/users/avatar/operator/67b2c5d990ba9800/resume-pic_1q3r11v.jpg|Zun Sien|Ling|member||online|False +operator|e486c123-0501-43e7-aa00-0231fb589c2e|potter2186@gmail.com||Harry|Wong|owner|Manager|away|False +operator|f43c636e-c023-4169-a555-be63a1f34a12|sookni98@gmail.com|https://storage.crisp.chat/users/avatar/operator/db5dc231ebfc1000/ksn-5_unmi7u.jpeg|Sook Ni|Kho|member||online|False +operator|fcbaf32c-29dc-4a24-af7d-77ceeb8616b9|yamzhenglim0427@gmail.com||ZhengLim|Yam|owner||online|False +sandbox|631dc1f5-f672-41a6-b32a-0dc05339c97f|yamzhenglim0427@gmail.com||ZhengLim|Yam|||| diff --git a/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc b/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc index fadded5f6046aba33f1ba19316aaea9ca8faff0c..c06031b5388fefe7f8a9336ff0aae0d8174244d8 100644 GIT binary patch delta 396 zcmbQj|C5(5k(ZZ?0SF%bu}D77wvo?`iH{%1x|`-U$CgK)%;>q(qx{@&nm_l z#a>vNUy_-ZK3SF3oKbXgENg)@b8>OnEw=Ka%#zfiDBi$O{}9)BUr#?zUq_$G?^z{y z`J*_31AXFMJe@;|BqocnHBDa2W@Bpr3>pST7A77b=3?Yv5@2Fs1k#LbKoJ2T6k`+t q@_Cp9fMP(Jnd`3*bCD`gk0wWv!ekwGO>S`@r-&Css7_8`Hvs^3(p1s_ delta 552 zcmZWm-D(p-6rM9X*;w1!-Wd=)8x;%A;rz_^ojKou`Qg0{{Ep{21W)p5tNYse=xw5pUxIWdlv{{2lqcOhn3Fu@ zHa|C~%9maqNq-JNb_gd7ZDIiFhZ|({B_Xi1U5x*+-_NpVb-2T;{KTCy6=d=Lc^ROT z$Nj0HLK)^4q+N>1Aip?Yj_u!OB&*Hf?At~SIme;X4rT<4@k>am_0Uma1v(E{WGuIrSo8Q1DmpR$_%z_wsT ze`A9yOr#xNSRx%9bh>F#ITF1rEsXByurQak(8J2A7q{)!{ku+kgr(zf;dMYb$hmOdNkRr6-^27VX;MqP9VY#LJ&FoTImh_;5l4K<7oxHLotv*G-V a4bi}tbz5B3Z_Il25?+K?hKBxZZvFz*xO?^h diff --git a/general_function.py b/general_function.py index bd7f0d9..f8ccf12 100644 --- a/general_function.py +++ b/general_function.py @@ -1,6 +1,7 @@ import json from operator import concat import os +import pandas as pd def read_json_file(path, name): try: @@ -24,4 +25,14 @@ def delete_csv_in_path(path): def change_to_current_directory(): script_dir = os.path.dirname(os.path.realpath(__file__)) - os.chdir(script_dir) \ No newline at end of file + os.chdir(script_dir) + + +def json_normalize_to_df(json): + try: + df = pd.json_normalize(json) + return df + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-json_normalize_to_df") + return False \ No newline at end of file diff --git a/main.py b/main.py index ca61cb5..7dd1229 100644 --- a/main.py +++ b/main.py @@ -5,6 +5,8 @@ import exchange.extract.exchange_extract_function as exchange_extract_func import exchange.load.exchange_load_function as exchange_load_func import shipping.extract.shipping_extract_function as shipping_extract_func import shipping.load.shipping_load_function as shipping_load_func +import crisp.extract.crisp_extract_function as crisp_extract_func + def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH): #Config file name @@ -40,8 +42,6 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH): file_path = os.path.join(dirpath,file_name) exchange_load_func.upload_to_bucket(ex_storage_client, file_name, file_path) - - def shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH): #Config file name SHIPPING_MYSQL_CONFIG_FILE = "SHIPPING_CONFIG.json" @@ -59,7 +59,6 @@ def shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH): #Run EX_SQL_DICT to get all data from mysql in csv format shipping_extract_func.mysql_dict_sql_to_csv(SHIPPING_SQL_DICT, SHIPPING_CSV_TEMP_STORAGE_PATH, SHIPPING_MYSQLDB, SHIPPING_MYSQLCURSOR) - def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH): #Config file name CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json" @@ -77,8 +76,25 @@ def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH): file_path = os.path.join(dirpath,file_name) shipping_load_func.upload_to_bucket(shipping_storage_client, file_name, file_path) +def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH): + #Config file name + CRISP_API_CONFIG_FILE = "CRISP_CONFIG.json" + CRISP_CSV_TEMP_STORAGE_PATH = CRISP_CSV_TEMP_STORAGE_PATH + #Read config file + CRISP_API_CONFIG = general_function.read_json_file("./config/",CRISP_API_CONFIG_FILE) + + #Get website_id + CRISP_WEBSITE_ID = CRISP_API_CONFIG['website_id'] + #Connect to CRISP + CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG) + + #Get operators csv + crisp_extract_func.get_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_operators.csv') + + # #Get users csv + crisp_extract_func.get_peoples_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_peoples.csv') def main(): #Make sure cron in the file directory @@ -104,6 +120,12 @@ def main(): general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH) + # #Make sure cron in the file directory + # general_function.change_to_current_directory() + + # #Extract Load Exchange Data + # CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/" + # crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH) diff --git a/requirements.txt b/requirements.txt index 15604fe..3e5b101 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ mysql-connector-python==8.0.29 -google-cloud-storage==2.5.0 \ No newline at end of file +google-cloud-storage==2.5.0 +crisp-api==1.1.13 \ No newline at end of file diff --git a/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc b/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb268101033c3bcf00d9372a520b17d62096bb14 GIT binary patch literal 1785 zcmZuxOK%%D5GJ`#J*_-k#Fy6SA%I#~^)U$2poL_k24WkD69jc+Y&T2FUT0q>sn}9> zb8)T(+J7J)`%C&Ch~9cCdTVb%3v@`yZW3e{91e%Ck2Bv)*lN`geA|B((XTE-zw5!f zAA!L)(8L`O6j7X@ech7;r(~btxy{b4c(7sC@>O${Pyob8pAFvzkbg^=y zJc~jZVQpA^7W5iLADS3}n0_JAF+L_G+_!>25+f}m?G;3wAMFa4_$k^$Hj>ul2aGDf zGW}HAB{|0!Nt?R5e~P{bgcW&BsJFM!*3mj>fXd%d_E9dRvd4mPcvZ$C8Ke_&l5BPz=(qB;HI}H{ky{_08R`iVLCk;?q+Pj z1{4-0d(64_V4GIn?Oa(*MWeC@zXPI~=kz$-_tvFAc%M5RArI zqyhDciE6#~77tK}TObf7*uDJ!aUB=8u@67L26K^i(B-FrH=vnt-iAPzxUzLXXQZ?* zaOISaxGmAl(jyZptV30znR9^#=-{s*soct&{x*G6S~Iuw^eWK23sm_fIrgg>C9_&t zE8PoH)#rpOt>gLto#U!e`m+XgWV39X;|m~#Y{_ESYe1f+TsrQQOfuzD3K$aI8xd>8&d($`do5iw;by z`n@C;GRo89{%sil4Gm4ZhS>p)f$l)|K;{8p=-o^5C``nCP%ihFI}|=uXj&V~(b}Cg z5g)>JUdB$7IAh9ZS(H;?=3898#h*hS)F?~hG?tK#wUco!S#%Ua>Sz^~eI)TtIV@ga zxI4Y^Y>=O3NgmSGKA=vRWCxqwWkqtK>>`v$%4KH&PbfbPUoaYT!8KKt1E-`ydF+fu zV+qJcOv2SkViI1tqnxLqR1L~R#N&DLx^V`XDU~C~aFeq0f@NI0AABiZD%U{cs`c>6 z?&FPMduwNF``ZVqHaF*|LOD^A!wI_db(fUC_wYe5*y=y#tNIMjxEYE;!zj_13n(HT z!-J7wM_?Xw>qAQriw*roRf8Iquq;fOvLH~4gE$l81YWW@-c0+Wc(KqsVj&PuC^9OP zn@%=ig0}(Bhtyd9C<~ThrC0}nuuYoK>ewe8Vq@sB3ziQ2pWvG?ZjuhzpttJ(Ubpzy zu&3+uFQHRDc#?-Q=W}l^nX7tNm6Yq6#6QzQ*B-O=9eBF@a6@1JUYgT!!tQIX3+-kf KH+9jn{PurI9q17N literal 0 HcmV?d00001 diff --git a/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc b/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b895fda1135b4bfcb139f227f69fd34150e8725a GIT binary patch literal 985 zcmaKrPfy!06u@mKDQ#ES2D|LAi5)u$#2HP~qyid;6$ysWN|5EoCJjyE%yAiw0vGl{ z;JB~k>rT_Y!FJhm8VcH^Im+{Uw)6h|9JaXFMle4A%!1zpq3_msIA9IEgEwCV!4bzX zO7H+<#0hu4q5SZcQb=NWNdsk*@?tbP9i*HETa(|6`oBkpA%>5}#ijlvU ziY*nL3x7-GV=Xhkx4XSVx0q%hSPD%`AQPX1xt849`6$Y=C=LBsGVb45)LQg(k_I-h z^=xWh{I_6>AYTH3Fu@(%z)QFZkLtimU^nkZ1C%+d5g5hm65RlQAmx5A8{Hglb7=Ujks$4#C6BH105#L-VXYR>3}vVv({$C|k1f zKy(|b36p8r!_@AXmOVu?rbp(nxCllp4Mpu!*>