diff --git a/EXCHANGE_EXTRACT_SQL_OLD.json b/EXCHANGE_EXTRACT_SQL_OLD.json
index 759c8f2..b8ddf9c 100644
--- a/EXCHANGE_EXTRACT_SQL_OLD.json
+++ b/EXCHANGE_EXTRACT_SQL_OLD.json
@@ -1,77 +1,82 @@
{
- "bookings" : "SELECT id, company_id, marking, service_id, bank_id, fix_amount, fix_currency_id, convertible_currency_id, conversion_currency_id, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM bookings;",
- "companies" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, reference, debtor, type, business_type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM companies;",
- "contacts" : "SELECT id, company_id, country_id, CONCAT(SUBSTR(reference,1,2),SUBSTR(reference,5,1),REPEAT('',LOCATE('@',reference)-3),SUBSTR(reference,LOCATE('@',reference)-1,3),REPEAT('',LENGTH(reference)-LOCATE('.',REVERSE(reference))-LOCATE('@',reference)-1),SUBSTR(reference,LENGTH(reference)-LOCATE('.',REVERSE(reference)))) AS reference, REPLACE(phone,RIGHT(phone,round(length(phone)/2,0)+1),REPEAT('*', round(length(phone)/2+0.5,0))) as phone, CONCAT(SUBSTR(email,1,2),SUBSTR(email,5,1),REPEAT('',LOCATE('@',email)-3),SUBSTR(email,LOCATE('@',email)-1,3),REPEAT('',LENGTH(email)-LOCATE('.',REVERSE(email))-LOCATE('@',email)-1),SUBSTR(email,LENGTH(email)-LOCATE('.',REVERSE(email)))) AS email, CONCAT(SUBSTR(wechat_id,1,2),SUBSTR(wechat_id,5,1),REPEAT('',LOCATE('@',wechat_id)-3),SUBSTR(wechat_id,LOCATE('@',wechat_id)-1,3),REPEAT('',LENGTH(wechat_id)-LOCATE('.',REVERSE(wechat_id))-LOCATE('@',wechat_id)-1),SUBSTR(wechat_id,LENGTH(wechat_id)-LOCATE('.',REVERSE(wechat_id)))) AS wechat_id, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM contacts;",
- "countries" : "SELECT *, now() AS '_source_loaded_at' FROM countries;",
- "currencies" : "SELECT *, now() AS '_source_loaded_at' FROM currencies;",
- "currency_rates" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rates;",
- "currency_rate_logs" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rate_logs;",
- "employees" : "SELECT *, now() AS '_source_loaded_at' FROM employees;",
- "groups" : "SELECT *, now() AS '_source_loaded_at' FROM groups;",
- "group_transactions" : "SELECT *, now() AS '_source_loaded_at' FROM group_transactions;",
- "segments" : "SELECT *, now() AS '_source_loaded_at' FROM segments;",
- "segment_companies" : "SELECT *, now() AS '_source_loaded_at' FROM segment_companies;",
- "service_types" : "SELECT *, now() AS '_source_loaded_at' FROM service_types;",
- "transactions" : "SELECT *, now() AS '_source_loaded_at' FROM transactions;",
- "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM `transaction_detail`;",
- "users" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, MD5(email) AS email_hash_id, type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM users;",
- "user_email_verifications" : "SELECT id, MD5(email) AS email_hash_id, is_complete, is_active, is_sent, created_at, updated_at, now() AS '_source_loaded_at' FROM user_email_verifications;",
- "wallets" : "SELECT id, owner_type, owner_id, CONCAT(SUBSTR(code,1,2),SUBSTR(code,5,1),REPEAT('',LOCATE('@',code)-3),SUBSTR(code,LOCATE('@',code)-1,3),REPEAT('',LENGTH(code)-LOCATE('.',REVERSE(code))-LOCATE('@',code)-1),SUBSTR(code,LENGTH(code)-LOCATE('.',REVERSE(code)))) AS code, currency_id, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM wallets;"
+ "activity_log" : "SELECT * FROM activity_log;",
+ "addresses" : "SELECT * FROM addresses;",
+ "announcements" : "SELECT * FROM announcements;",
+ "announcement_segment" : "SELECT * FROM announcement_segment;",
+ "banks" : "SELECT * FROM banks;",
+ "bank_logs" : "SELECT * FROM bank_logs;",
+ "bookings" : "SELECT * FROM bookings;",
+ "companies" : "SELECT * FROM companies;",
+ "contacts" : "SELECT * FROM contacts;",
+ "countries" : "SELECT * FROM countries;",
+ "currencies" : "SELECT * FROM currencies;",
+ "currency_logs" : "SELECT * FROM currency_logs;",
+ "currency_rates" : "SELECT * FROM currency_rates;",
+ "currency_rate_logs" : "SELECT * FROM currency_rate_logs;",
+ "districts" : "SELECT * FROM districts;",
+ "documents" : "SELECT * FROM documents;",
+ "employees" : "SELECT * FROM employees;",
+ "failed_jobs" : "SELECT * FROM failed_jobs;",
+ "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '
'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
+ "groups" : "SELECT * FROM groups;",
+ "group_transactions" : "SELECT * FROM group_transactions;",
+ "jobs" : "SELECT * FROM jobs;",
+ "migrations" : "SELECT * FROM migrations;",
+ "model_has_roles" : "SELECT * FROM model_has_roles;",
+ "notifications" : "SELECT * FROM notifications;",
+ "password_resets" : "SELECT * FROM password_resets;",
+ "permissions" : "SELECT * FROM permissions;",
+ "roles" : "SELECT * FROM roles;",
+ "role_has_permissions" : "SELECT * FROM role_has_permissions;",
+ "segments" : "SELECT * FROM segments;",
+ "segment_companies" : "SELECT * FROM segment_companies;",
+ "segment_constants" : "SELECT * FROM segment_constants;",
+ "service_types" : "SELECT * FROM service_types;",
+ "states" : "SELECT * FROM states;",
+ "transactions" : "SELECT * FROM transactions;",
+ "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', '
'),'|', '|'),'\\\\','\\') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', '
'),'|', '|'),'\\\\','\\') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;",
+ "users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
+ "user_email_verifications" : "SELECT * FROM user_email_verifications;",
+ "wallets" : "SELECT * FROM wallets;"
}
+
+
{
"activity_log" : "SELECT * FROM activity_log;",
"addresses" : "SELECT * FROM addresses;",
"announcements" : "SELECT * FROM announcements;",
"announcement_segment" : "SELECT * FROM announcement_segment;",
- // "banks" : "SELECT * FROM banks;",
- // "bank_logs" : "SELECT * FROM bank_logs;",
- // "bookings" : "SELECT * FROM bookings;",
"companies" : "SELECT * FROM companies;",
- --"company_connections" : "SELECT * FROM company_connections;",
- --"company_employees" : "SELECT * FROM company_employees;",
- --"company_modules" : "SELECT * FROM company_modules;",
- --"connection_segments" : "SELECT * FROM connection_segments;",
+ "company_connections" : "SELECT * FROM company_connections;",
+ "company_employees" : "SELECT * FROM company_employees;",
+ "company_modules" : "SELECT * FROM company_modules;",
+ "connection_segments" : "SELECT * FROM connection_segments;",
"contacts" : "SELECT * FROM contacts;",
- --"containers" : "SELECT * FROM countries",
- --"container_packing_lists" : "SELECT * FROM container_packing_lists",
+ "containers" : "SELECT * FROM containers",
+ "container_packing_lists" : "SELECT * FROM container_packing_lists",
"countries" : "SELECT * FROM countries;",
"currencies" : "SELECT * FROM currencies;",
- // "currency_logs" : "SELECT * FROM currency_logs;",
- // "currency_rates" : "SELECT * FROM currency_rates;",
- // "currency_rate_logs" : "SELECT * FROM currency_rate_logs;",
"districts" : "SELECT * FROM districts;",
"documents" : "SELECT * FROM documents;",
- // "employees" : "SELECT * FROM employees;",
"failed_jobs" : "SELECT * FROM failed_jobs;",
- "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', ''),'|', ''),'\\\\','') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
- // "groups" : "SELECT * FROM groups;",
- // "group_transactions" : "SELECT * FROM group_transactions;",
- "jobs" : "SELECT * FROM jobs;",
+ "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '
'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
"migrations" : "SELECT * FROM migrations;",
- // "model_has_roles" : "SELECT * FROM model_has_roles;",
- // "notifications" : "SELECT * FROM notifications;",
- --"orders" : "SELECT * FROM orders",
- --"order_roles" : "SELECT * FROM order_roles",
- --"packages" : "SELECT * FROM packages",
- --"packing_lists" : "SELECT * FROM packing_lists",
+ "orders" : "SELECT * FROM orders",
+ "order_roles" : "SELECT * FROM order_roles",
+ "packages" : "SELECT * FROM packages",
+ "packing_lists" : "SELECT * FROM packing_lists",
"password_resets" : "SELECT * FROM password_resets;",
- // "permissions" : "SELECT * FROM permissions;",
- --"remarks" : "SELECT * FROM remarks",
- // "roles" : "SELECT * FROM roles;",
- // "role_has_permissions" : "SELECT * FROM role_has_permissions;",
- --"schedules" : "SELECT * FROM schedules",
+ "remarks" : "SELECT id, owner_type, owner_id, commenter_id, REPLACE(REPLACE(REPLACE(content, '\n', '
'),'|', '|'),'\\\\','\\') AS 'content', deleted_at, created_at, updated_at FROM remarks",
+ "schedules" : "SELECT * FROM schedules",
"segments" : "SELECT * FROM segments;",
- // "segment_companies" : "SELECT * FROM segment_companies;",
"segment_constants" : "SELECT * FROM segment_constants;",
- // "service_types" : "SELECT * FROM service_types;",
"states" : "SELECT * FROM states;",
- --"steps" : "SELECT * FROM steps",
+ "steps" : "SELECT * FROM steps",
"transactions" : "SELECT * FROM transactions;",
- "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;",
- --"transport_arrangements" : "SELECT * FROM transport_arrangements",
+ "transaction_details" : "SELECT * FROM `transaction_details`;",
+ "transport_arrangements" : "SELECT * FROM transport_arrangements",
"users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
- "user_email_verifications" : "SELECT * FROM user_email_verifications;",
- // "wallets" : "SELECT * FROM wallets;"
-}
\ No newline at end of file
+ "user_email_verifications" : "SELECT * FROM user_email_verifications;"
+}
diff --git a/__pycache__/general_function.cpython-39.pyc b/__pycache__/general_function.cpython-39.pyc
index 710ebe6..8e8548e 100644
Binary files a/__pycache__/general_function.cpython-39.pyc and b/__pycache__/general_function.cpython-39.pyc differ
diff --git a/config/CRISP_CONFIG.json b/config/CRISP_CONFIG.json
new file mode 100644
index 0000000..601b1e2
--- /dev/null
+++ b/config/CRISP_CONFIG.json
@@ -0,0 +1,5 @@
+{
+ "identifier" : "0f8edfb8-1271-4e8c-bdfc-e33de84a0efe",
+ "key" : "408094e63ab670c968b3f9a119387263d4e23721817513b47bd94700bf4349a9",
+ "website_id" : "665dcd41-1edf-4451-8cb9-f1cf9ed35e15"
+}
\ No newline at end of file
diff --git a/config/EXCHANGE_EXTRACT_SQL.json b/config/EXCHANGE_EXTRACT_SQL.json
index f39a497..86ae5a9 100644
--- a/config/EXCHANGE_EXTRACT_SQL.json
+++ b/config/EXCHANGE_EXTRACT_SQL.json
@@ -1,43 +1,43 @@
{
- "activity_log" : "SELECT * FROM activity_log;",
- "addresses" : "SELECT * FROM addresses;",
+ "activity_log" : "SELECT ID, LOG_NAME, DESCRIPTION, SUBJECT_ID, SUBJECT_TYPE, CAUSER_ID, CAUSER_TYPE, '**MASKING_PROPERTIES**' AS PROPERTIES, CREATED_AT, UPDATED_AT FROM activity_log;",
+ "addresses" : "SELECT ID, COMPANY_ID, COUNTRY_ID, STATE_ID, DISTRICT_ID, POSTCODE, '**MASKING_STREET_ONE**'AS STREET_ONE, '**MASKING_STREET_TWO**'AS STREET_TWO, BILLING, DELETED_AT, CREATED_AT, UPDATED_AT FROM addresses;",
"announcements" : "SELECT * FROM announcements;",
"announcement_segment" : "SELECT * FROM announcement_segment;",
- "banks" : "SELECT * FROM banks;",
- "bank_logs" : "SELECT * FROM bank_logs;",
+ "banks" : "SELECT ID, COMPANY_ID, '**MASKING_REFERENCE**' AS REFERENCE, BANK_NAME, MD5(CONCAT(HOLDER_NAME,LENGTH(HOLDER_NAME)*2022*2)) AS HOLDER_NAME, MD5(CONCAT(ACCOUNT_NO,LENGTH(ACCOUNT_NO)*2022*2)) AS ACCOUNT_NO, BANK_BRANCH, MD5(CONCAT(SWIFT,LENGTH(SWIFT)*2022*2)) AS SWIFT, MD5(CONCAT(SNAP,LENGTH(SNAP)*2022*2)) AS SNAP, TYPE, DEFAULT, STATUS, COUNTRY_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM banks;",
+ "bank_logs" : "SELECT ID, BANK_ID, COMPANY_ID, '**MASKING_REFERENCE**' AS REFERENCE, BANK_NAME, MD5(CONCAT(HOLDER_NAME,LENGTH(HOLDER_NAME)*2022*2)) AS HOLDER_NAME, MD5(CONCAT(ACCOUNT_NO,LENGTH(ACCOUNT_NO)*2022*2)) AS ACCOUNT_NO, BANK_BRANCH, MD5(CONCAT(SWIFT,LENGTH(SWIFT)*2022*2)) AS SWIFT, MD5(CONCAT(SNAP,LENGTH(SNAP)*2022*2)) AS SNAP, TYPE, DEFAULT, STATUS, COUNTRY_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM bank_logs;",
"bookings" : "SELECT * FROM bookings;",
- "companies" : "SELECT * FROM companies;",
- "contacts" : "SELECT * FROM contacts;",
+ "companies" : "SELECT ID, '**MASKING_NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, DEBTOR, TYPE, BUSINESS_TYPE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM companies;",
+ "contacts" : "SELECT ID, COMPANY_ID, COUNTRY_ID, '**MASKING_REFERENCE**' AS REFERENCE, MD5(CONCAT(PHONE,LENGTH(PHONE)*2022*2)) AS PHONE, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, MD5(CONCAT(WECHAT_ID,LENGTH(WECHAT_ID)*2022*2)) AS WECHAT_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM contacts;",
"countries" : "SELECT * FROM countries;",
"currencies" : "SELECT * FROM currencies;",
"currency_logs" : "SELECT * FROM currency_logs;",
"currency_rates" : "SELECT * FROM currency_rates;",
"currency_rate_logs" : "SELECT * FROM currency_rate_logs;",
"districts" : "SELECT * FROM districts;",
- "documents" : "SELECT * FROM documents;",
+ "documents" : "SELECT ID, OWNER_TYPE, OWNER_ID, DOCUMENT_TYPE, '**MASKING_REFERENCE**' AS REFERENCE, STATUS, ISSUED_DATE, EXPIRED_DATE, APPROVER, APPROVAL_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM documents;",
"employees" : "SELECT * FROM employees;",
- "failed_jobs" : "SELECT * FROM failed_jobs;",
- "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '
'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
- "groups" : "SELECT * FROM groups;",
+ "failed_jobs" : "SELECT ID, CONNECTION, QUEUE, '**MASKING_PAYLOAD' AS PAYLOAD, '**MASKING_EXCEPTION**' AS EXCEPTION, FAILED_AT FROM failed_jobs;",
+ "files" : "SELECT id, document_id, '**MASKING_FILE**', file_type, deleted_at, created_at, updated_at FROM files;",
+ "groups" : "SELECT ID, '**MASKING_REFERENCE**' AS REFERENCE, ISSUER, RECEIVER, AMOUNT, ORIGINAL_AMOUNT, CURRENCY_ID, ORIGINAL_CURRENCY_ID, CURRENCY_RATE, TAX, SERVICE_CHARGE, STATUS, CREATED_AT, UPDATED_AT FROM groups;",
"group_transactions" : "SELECT * FROM group_transactions;",
- "jobs" : "SELECT * FROM jobs;",
+ "jobs" : "SELECT ID, QUEUE, '**MASKING_PAYLOAD**' AS PAYLOAD, ATTEMPTS, RESERVED_AT, AVAILABLE_AT, CREATED_AT FROM jobs;",
"migrations" : "SELECT * FROM migrations;",
"model_has_roles" : "SELECT * FROM model_has_roles;",
"notifications" : "SELECT * FROM notifications;",
- "password_resets" : "SELECT * FROM password_resets;",
+ "password_resets" : "SELECT ID, USER_ID, '**MASKING_TOKEN**' AS TOKEN, IS_EXPIRED, IS_COMPLETE, CREATED_AT, UPDATED_AT FROM password_resets;",
"permissions" : "SELECT * FROM permissions;",
"roles" : "SELECT * FROM roles;",
"role_has_permissions" : "SELECT * FROM role_has_permissions;",
"segments" : "SELECT * FROM segments;",
"segment_companies" : "SELECT * FROM segment_companies;",
- "segment_constants" : "SELECT * FROM segment_constants;",
+ "segment_constants" : "SELECT ID, SEGMENT_ID, NAME, REFERENCE, '**MASKING_DETAIL**' AS DETAIL, DELETED_AT, CREATED_AT, UPDATED_AT FROM segment_constants;",
"service_types" : "SELECT * FROM service_types;",
"states" : "SELECT * FROM states;",
- "transactions" : "SELECT * FROM transactions;",
+ "transactions" : "SELECT ID, OWNER_TYPE, OWNER_ID, TYPE, ISSUER, RECEIVER, RECIPIENT_BANK_ACCOUNT_ID, PAYMENT_METHOD, '**PAYMENT_REFERENCE**' AS PAYMENT_REFERENCE, MD5(CONCAT(BILL_NO,LENGTH(BILL_NO)*2022*2)) AS BILL_NO, AMOUNT, ORIGINAL_AMOUNT, CURRENCY_ID, ORIGINAL_CURRENCY_ID, CURRENCY_RATE, TAX, SERVICE_CHARGE, EXPIRES_ON, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM transactions;",
"transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', '
'),'|', '|'),'\\\\','\\') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', '
'),'|', '|'),'\\\\','\\') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;",
- "users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
- "user_email_verifications" : "SELECT * FROM user_email_verifications;",
- "wallets" : "SELECT * FROM wallets;"
+ "users" : "SELECT id, '**MASKING_NAME**' AS name, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
+ "user_email_verifications" : "SELECT ID, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, '**MASKING_TOKEN**' AS TOKEN, IS_COMPLETE, IS_ACTIVE, IS_SENT, CREATED_AT, UPDATED_AT FROM user_email_verifications;",
+ "wallets" : "SELECT ID, OWNER_TYPE, OWNER_ID, MD5(CONCAT(CODE,LENGTH(CODE)*2022*2)) AS CODE, CURRENCY_ID, AMOUNT, DELETED_AT, CREATED_AT, UPDATED_AT FROM wallets;"
}
diff --git a/config/SHIPPING_EXTRACT_SQL.json b/config/SHIPPING_EXTRACT_SQL.json
index d49fbd9..a9deba7 100644
--- a/config/SHIPPING_EXTRACT_SQL.json
+++ b/config/SHIPPING_EXTRACT_SQL.json
@@ -1,37 +1,37 @@
{
- "activity_log" : "SELECT * FROM activity_log;",
- "addresses" : "SELECT * FROM addresses;",
+ "activity_log" : "SELECT ID, LOG_NAME, DESCRIPTION, SUBJECT_TYPE, SUBJECT_ID, CAUSER_TYPE, CAUSER_ID, '**MASKING_PROPERTIES**' AS PROPERTIES, CREATED_AT, UPDATED_AT FROM activity_log;",
+ "addresses" : "SELECT ID, OWNER_TYPE, OWNER_ID, '**MASKING_REFERENCE**' AS REFERENCE, '**MASKING_STREET_ONE**' AS STREET_ONE, '**MASKING_STREET_TWO**'AS STREET_TWO, DISTRICT_ID, STATE_ID, COUNTRY_ID, POSTCODE, TYPE, DEFAULT, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM addresses;",
"announcements" : "SELECT * FROM announcements;",
"announcement_segment" : "SELECT * FROM announcement_segment;",
- "companies" : "SELECT * FROM companies;",
- "company_connections" : "SELECT * FROM company_connections;",
+ "companies" : "SELECT ID, '**MASKING_NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, DEBTOR, TYPE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM companies;",
+ "company_connections" : "SELECT ID, INVITER_ID, INVITEE_ID, '**MASKING INVITER_REFERENCE**' AS INVITER_REFERENCE, '**MASKING INVITEE_REFERENCE**' AS INVITEE_REFERENCE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM company_connections;",
"company_employees" : "SELECT * FROM company_employees;",
- "company_modules" : "SELECT * FROM company_modules;",
+ "company_modules" : "SELECT ID, COMPANY_ID, '**MASKING NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, TYPE, STATUS, MD5(CONCAT(UNITY_HASH_ID,LENGTH(UNITY_HASH_ID)*2022*2)) AS UNITY_HASH_ID, MD5(CONCAT(UNITY_SIGNATURE,LENGTH(UNITY_SIGNATURE)*2022*2)) AS UNITY_SIGNATURE, DELETED_AT, CREATED_AT, UPDATED_AT FROM company_modules;",
"connection_segments" : "SELECT * FROM connection_segments;",
- "contacts" : "SELECT * FROM contacts;",
+ "contacts" : "SELECT ID, OWNER_TYPE, OWNER_ID, '**MASKING_REFERENCE**' AS REFERENCE, MD5(CONCAT(PHONE,LENGTH(PHONE)*2022*2)) AS PHONE, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, MD5(CONCAT(WECHAT_ID,LENGTH(WECHAT_ID)*2022*2)) AS WECHAT_ID, DEFAULT, DELETED_AT, CREATED_AT, UPDATED_AT FROM contacts;",
"containers" : "SELECT * FROM containers",
"container_packing_lists" : "SELECT * FROM container_packing_lists",
"countries" : "SELECT * FROM countries;",
"currencies" : "SELECT * FROM currencies;",
"districts" : "SELECT * FROM districts;",
- "documents" : "SELECT * FROM documents;",
- "failed_jobs" : "SELECT * FROM failed_jobs;",
- "files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '
'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
+ "documents" : "SELECT ID, OWNER_TYPE, OWNER_ID, DOCUMENT_TYPE, '**MASKING_REFERENCE**' AS REFERENCE, STATUS, ISSUED_DATE, EXPIRED_DATE, APPROVER, APPROVAL_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM documents;",
+ "failed_jobs" : "SELECT ID, UUID, CONNECTION, QUEUE, '**MASKING_PAYLOAD' AS PAYLOAD, '**MASKING_EXCEPTION**' AS EXCEPTION, FAILED_AT FROM failed_jobs;",
+ "files" : "SELECT id, document_id, '**MASKING FILE**' AS FILE, file_type, deleted_at, created_at, updated_at FROM files;",
"migrations" : "SELECT * FROM migrations;",
"orders" : "SELECT * FROM orders",
- "order_roles" : "SELECT * FROM order_roles",
+ "order_roles" : "SELECT ID, ORDER_ID, COMPANY_MODULE_ID, ROLE_ID, MD5(CONCAT(ENTITY_HASH_ID,LENGTH(ENTITY_HASH_ID)*2022*2)) AS ENTITY_HASH_ID. MD5(CONCAT(ENTITY_SIGNATURE,LENGTH(ENTITY_SIGNATURE)*2022*2)) AS ENTITY_SIGNATURE, DELETED_AT, CREATED_AT, UPDATED_AT FROM order_roles",
"packages" : "SELECT * FROM packages",
"packing_lists" : "SELECT * FROM packing_lists",
- "password_resets" : "SELECT * FROM password_resets;",
- "remarks" : "SELECT id, owner_type, owner_id, commenter_id, REPLACE(REPLACE(REPLACE(content, '\n', '
'),'|', '|'),'\\\\','\\') AS 'content', deleted_at, created_at, updated_at FROM remarks",
+ "password_resets" : "SELECT ID, USER_ID, '**MASKING_TOKEN**' AS TOKEN, IS_EXPIRED, IS_COMPLETE, CREATED_AT, UPDATED_AT FROM password_resets;",
+ "remarks" : "SELECT id, owner_type, owner_id, commenter_id, '**MASKING CONTENT**' AS CONTENT, deleted_at, created_at, updated_at FROM remarks",
"schedules" : "SELECT * FROM schedules",
"segments" : "SELECT * FROM segments;",
"segment_constants" : "SELECT * FROM segment_constants;",
"states" : "SELECT * FROM states;",
- "steps" : "SELECT * FROM steps",
+ "steps" : "SELECT ID, OWNER_TYPE, OWNER_ID, APPOINTEE_ID, MD5(CONCAT(OBLIGATION_HASH_ID,LENGTH(OBLIGATION_HASH_ID)*2022*2)) AS OBLIGATION_HASH_ID, REFERENCE, PRIMARY, SEQUENCE, STATUS, COMPLETE_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM steps",
"transactions" : "SELECT * FROM transactions;",
"transaction_details" : "SELECT * FROM `transaction_details`;",
- "transport_arrangements" : "SELECT * FROM transport_arrangements",
- "users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
- "user_email_verifications" : "SELECT * FROM user_email_verifications;"
+ "transport_arrangements" : "SELECT ID, OWNER_TYPE, OWNER_ID, TYPE, COURIER, LEFT(TRACKING_NUMBER,6) AS TRACKING_NUMBER, DISPATCH_DATE, DROP_DATE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM transport_arrangements",
+ "users" : "SELECT id, '**MASKING NAME**' AS NAME, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
+ "user_email_verifications" : "SELECT ID, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, '**MASKING_TOKEN**' AS TOKEN, IS_COMPLETE, IS_ACTIVE, IS_SENT, CREATED_AT, UPDATED_AT FROM user_email_verifications;"
}
diff --git a/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc b/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc
new file mode 100644
index 0000000..2e86a95
Binary files /dev/null and b/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc differ
diff --git a/crisp/extract/crisp_extract_function.py b/crisp/extract/crisp_extract_function.py
new file mode 100644
index 0000000..735e888
--- /dev/null
+++ b/crisp/extract/crisp_extract_function.py
@@ -0,0 +1,61 @@
+from crisp_api import Crisp
+import general_function
+import pandas as pd
+
+def connect_crisp(config):
+ try:
+ client = Crisp()
+ client.set_tier("plugin")
+ client.authenticate(config["identifier"], config["key"])
+ return client
+ except Exception as e:
+ print(e)
+ print("[FUNCTION_ERROR]-connect_crisp")
+ return False
+
+def get_operators_csv(website_id, client, path, filename):
+ try:
+ output_location = path + filename
+ operators_list_json = client.website.list_website_operators(website_id)
+ operators_list_df = general_function.json_normalize_to_df(operators_list_json)
+ operators_list_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
+ print("[Download Complete]",filename)
+ except Exception as e:
+ print(e)
+ print("[FUNCTION_ERROR]-get_operators_csv")
+ return False
+
+def get_peoples_csv(website_id, client, path, filename):
+ try:
+ output_location = path + filename
+ peoples_list_df = pd.DataFrame()
+ peoples_data_df = pd.DataFrame()
+ loop_flag = True
+ page_number = 1
+
+ while loop_flag == True:
+ peoples_list_json = client.website.list_people_profiles(website_id, page_number)
+ temp_df = general_function.json_normalize_to_df(peoples_list_json)
+ peoples_list_df = pd.concat([temp_df,peoples_list_df], ignore_index=True)
+ if not peoples_list_json:
+ loop_flag = False
+ else:
+ page_number += 1
+
+ for people_id in peoples_list_df.people_id:
+ print(people_id)
+ people_data_json = client.website.get_people_data(website_id, people_id)
+ temp_df = general_function.json_normalize_to_df(people_data_json)
+ temp_df['people_id'] = people_id
+ peoples_data_df = pd.concat([temp_df,peoples_data_df], ignore_index=True)
+
+ peoples_list_df = peoples_list_df.merge(peoples_data_df, on='people_id', how='left')
+
+ peoples_list_df.to_csv(output_location, sep=',', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
+ peoples_data_df.to_csv('test.csv', sep=',', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
+ print("[Download Complete]",filename)
+
+ except Exception as e:
+ print(e)
+ print("[FUNCTION_ERROR]-get_peoples_csv")
+ return False
\ No newline at end of file
diff --git a/crisp/extract/crisp_temp_extract_csv/.gitkeep b/crisp/extract/crisp_temp_extract_csv/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/crisp/extract/crisp_temp_extract_csv/crisp_operators.csv b/crisp/extract/crisp_temp_extract_csv/crisp_operators.csv
new file mode 100644
index 0000000..ccf4889
--- /dev/null
+++ b/crisp/extract/crisp_temp_extract_csv/crisp_operators.csv
@@ -0,0 +1,20 @@
+type|details.user_id|details.email|details.avatar|details.first_name|details.last_name|details.role|details.title|details.availability|details.has_token
+operator|15f0ec35-8a0c-457f-83ad-68e04f3b6624|taypeisongtps@gmail.com|https://storage.crisp.chat/users/avatar/operator/5e71c78373d9ac00/potrait-photo_gsa6gd.jpg|Pei Song|Tay|member|Customer Service|away|False
+operator|293073d9-e65f-40e2-a1ca-54fb9a2e4ff7|ykheng2bb@gmail.com|https://storage.crisp.chat/users/avatar/operator/4a7ce09c05d7b400/whatsapp-image-2022-04-13-at-2_19jwcch.jpeg|YAP|KAH HENG|owner|Customer Service|online|False
+operator|294351f3-d501-41dd-a095-2c512dd2ff4f|ciefworldwide.acc1@gmail.com||Haizat|Zaidi|owner||away|False
+operator|2a0b5503-97ff-4875-918d-be8185bc58a1|h2009413007@hcu.edu.my|https://storage.crisp.chat/users/avatar/operator/6bba6be706925c00/screenshot-2022-09-30-150004_mst2g.png|Tok|soon wee|member|Consultant|online|False
+operator|413c155c-3f3b-4859-bc18-dddb17dfd3ac|diliniwr10@gmail.com|https://storage.crisp.chat/users/avatar/operator/ed6e6534f8f01000/dili_wntzti.png|Dilini|WR|member||away|False
+operator|6f231d14-b3c0-47b5-bd02-617579cd2bcb|kangjun8331@gmail.com|https://storage.crisp.chat/users/avatar/operator/9ccd7bf346431800/img1413_d7cf1m.jpg|Kang|Jun|member|Consultant|away|False
+operator|80708bf4-9c08-4b8d-a37b-3a2f3c9570f1|jhunyanwong@gmail.com|https://storage.crisp.chat/users/avatar/operator/b20140b4cfccf800/214025936361797548903332433376_1r1o1en.jpg|JY|Wong|owner||away|False
+operator|83fb3451-f17a-4876-b317-244c30b27c8b|shafiqa_sukeri@cief-malaysia.com|https://storage.crisp.chat/users/avatar/operator/6ba11d171dddb800/photo2022-07-0811-15-15_9ulegi.jpg|FAIN|BACKEND|owner|Customer Service|online|False
+operator|99a1c001-60ac-4f71-91e1-33e8c2205525|enxin1227@gmail.com|https://storage.crisp.chat/users/upload/operator/bb7c7077d72bc800/21064570005_zf9jf6.jpg|Tiffany|Ooi|member||online|False
+operator|9df5d57c-8eb6-4e43-a8c5-cc6282564de9|ciefworldwide@gmail.com||PM|Wong|owner|CEO|away|False
+operator|a8079229-94c0-4a33-a027-bcbaed17a1c0|uldvstar@gmail.com|https://storage.crisp.chat/users/avatar/operator/8f0372fcb5eb0000/download-1_17y48zb.png|Omair|Saleh|owner||away|False
+operator|d8e200bc-709b-4b80-a706-bab8ec612e0b|infociefworldwide@gmail.com||CIEF|Marketing|owner|Marketing|away|False
+operator|d9ef0024-60c9-4137-919d-1d25bc2fab0e|operationteam@cief-malaysia.com||Operation|Team|member||away|False
+operator|db57d7a6-9421-43b9-9614-0867db22739a|ziwen38@gmail.com|https://storage.crisp.chat/users/avatar/operator/eed9c8d96123e00/resume_1x03t8n.jpeg|Zi Wen|Lee|member||online|False
+operator|dd8d2dcb-07a4-4343-b5bd-93038d08bd54|lingzunsien1211@gmail.com|https://storage.crisp.chat/users/avatar/operator/67b2c5d990ba9800/resume-pic_1q3r11v.jpg|Zun Sien|Ling|member||online|False
+operator|e486c123-0501-43e7-aa00-0231fb589c2e|potter2186@gmail.com||Harry|Wong|owner|Manager|away|False
+operator|f43c636e-c023-4169-a555-be63a1f34a12|sookni98@gmail.com|https://storage.crisp.chat/users/avatar/operator/db5dc231ebfc1000/ksn-5_unmi7u.jpeg|Sook Ni|Kho|member||online|False
+operator|fcbaf32c-29dc-4a24-af7d-77ceeb8616b9|yamzhenglim0427@gmail.com||ZhengLim|Yam|owner||online|False
+sandbox|631dc1f5-f672-41a6-b32a-0dc05339c97f|yamzhenglim0427@gmail.com||ZhengLim|Yam||||
diff --git a/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc b/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc
index fadded5..c06031b 100644
Binary files a/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc and b/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc differ
diff --git a/general_function.py b/general_function.py
index bd7f0d9..f8ccf12 100644
--- a/general_function.py
+++ b/general_function.py
@@ -1,6 +1,7 @@
import json
from operator import concat
import os
+import pandas as pd
def read_json_file(path, name):
try:
@@ -24,4 +25,14 @@ def delete_csv_in_path(path):
def change_to_current_directory():
script_dir = os.path.dirname(os.path.realpath(__file__))
- os.chdir(script_dir)
\ No newline at end of file
+ os.chdir(script_dir)
+
+
+def json_normalize_to_df(json):
+ try:
+ df = pd.json_normalize(json)
+ return df
+ except Exception as e:
+ print(e)
+ print("[FUNCTION_ERROR]-json_normalize_to_df")
+ return False
\ No newline at end of file
diff --git a/main.py b/main.py
index ca61cb5..7dd1229 100644
--- a/main.py
+++ b/main.py
@@ -5,6 +5,8 @@ import exchange.extract.exchange_extract_function as exchange_extract_func
import exchange.load.exchange_load_function as exchange_load_func
import shipping.extract.shipping_extract_function as shipping_extract_func
import shipping.load.shipping_load_function as shipping_load_func
+import crisp.extract.crisp_extract_function as crisp_extract_func
+
def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
#Config file name
@@ -40,8 +42,6 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH):
file_path = os.path.join(dirpath,file_name)
exchange_load_func.upload_to_bucket(ex_storage_client, file_name, file_path)
-
-
def shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH):
#Config file name
SHIPPING_MYSQL_CONFIG_FILE = "SHIPPING_CONFIG.json"
@@ -59,7 +59,6 @@ def shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH):
#Run EX_SQL_DICT to get all data from mysql in csv format
shipping_extract_func.mysql_dict_sql_to_csv(SHIPPING_SQL_DICT, SHIPPING_CSV_TEMP_STORAGE_PATH, SHIPPING_MYSQLDB, SHIPPING_MYSQLCURSOR)
-
def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
#Config file name
CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
@@ -77,8 +76,25 @@ def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
file_path = os.path.join(dirpath,file_name)
shipping_load_func.upload_to_bucket(shipping_storage_client, file_name, file_path)
+def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
+ #Config file name
+ CRISP_API_CONFIG_FILE = "CRISP_CONFIG.json"
+ CRISP_CSV_TEMP_STORAGE_PATH = CRISP_CSV_TEMP_STORAGE_PATH
+ #Read config file
+ CRISP_API_CONFIG = general_function.read_json_file("./config/",CRISP_API_CONFIG_FILE)
+
+ #Get website_id
+ CRISP_WEBSITE_ID = CRISP_API_CONFIG['website_id']
+ #Connect to CRISP
+ CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG)
+
+ #Get operators csv
+ crisp_extract_func.get_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_operators.csv')
+
+ # #Get users csv
+ crisp_extract_func.get_peoples_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_peoples.csv')
def main():
#Make sure cron in the file directory
@@ -104,6 +120,12 @@ def main():
general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
+ # #Make sure cron in the file directory
+ # general_function.change_to_current_directory()
+
+ # #Extract Load Exchange Data
+ # CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
+ # crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
diff --git a/requirements.txt b/requirements.txt
index 15604fe..3e5b101 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
mysql-connector-python==8.0.29
-google-cloud-storage==2.5.0
\ No newline at end of file
+google-cloud-storage==2.5.0
+crisp-api==1.1.13
\ No newline at end of file
diff --git a/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc b/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc
new file mode 100644
index 0000000..bb26810
Binary files /dev/null and b/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc differ
diff --git a/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc b/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc
new file mode 100644
index 0000000..b895fda
Binary files /dev/null and b/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc differ