mirror of
https://gitlab.com/cief-data/data-management.git
synced 2026-08-19 04:13:57 +00:00
ETLT and CRISP
This commit is contained in:
Yam ZhengLim
@@ -1,77 +1,82 @@
|
||||
{
|
||||
"bookings" : "SELECT id, company_id, marking, service_id, bank_id, fix_amount, fix_currency_id, convertible_currency_id, conversion_currency_id, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM bookings;",
|
||||
"companies" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, reference, debtor, type, business_type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM companies;",
|
||||
"contacts" : "SELECT id, company_id, country_id, CONCAT(SUBSTR(reference,1,2),SUBSTR(reference,5,1),REPEAT('',LOCATE('@',reference)-3),SUBSTR(reference,LOCATE('@',reference)-1,3),REPEAT('',LENGTH(reference)-LOCATE('.',REVERSE(reference))-LOCATE('@',reference)-1),SUBSTR(reference,LENGTH(reference)-LOCATE('.',REVERSE(reference)))) AS reference, REPLACE(phone,RIGHT(phone,round(length(phone)/2,0)+1),REPEAT('*', round(length(phone)/2+0.5,0))) as phone, CONCAT(SUBSTR(email,1,2),SUBSTR(email,5,1),REPEAT('',LOCATE('@',email)-3),SUBSTR(email,LOCATE('@',email)-1,3),REPEAT('',LENGTH(email)-LOCATE('.',REVERSE(email))-LOCATE('@',email)-1),SUBSTR(email,LENGTH(email)-LOCATE('.',REVERSE(email)))) AS email, CONCAT(SUBSTR(wechat_id,1,2),SUBSTR(wechat_id,5,1),REPEAT('',LOCATE('@',wechat_id)-3),SUBSTR(wechat_id,LOCATE('@',wechat_id)-1,3),REPEAT('',LENGTH(wechat_id)-LOCATE('.',REVERSE(wechat_id))-LOCATE('@',wechat_id)-1),SUBSTR(wechat_id,LENGTH(wechat_id)-LOCATE('.',REVERSE(wechat_id)))) AS wechat_id, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM contacts;",
|
||||
"countries" : "SELECT *, now() AS '_source_loaded_at' FROM countries;",
|
||||
"currencies" : "SELECT *, now() AS '_source_loaded_at' FROM currencies;",
|
||||
"currency_rates" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rates;",
|
||||
"currency_rate_logs" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rate_logs;",
|
||||
"employees" : "SELECT *, now() AS '_source_loaded_at' FROM employees;",
|
||||
"groups" : "SELECT *, now() AS '_source_loaded_at' FROM groups;",
|
||||
"group_transactions" : "SELECT *, now() AS '_source_loaded_at' FROM group_transactions;",
|
||||
"segments" : "SELECT *, now() AS '_source_loaded_at' FROM segments;",
|
||||
"segment_companies" : "SELECT *, now() AS '_source_loaded_at' FROM segment_companies;",
|
||||
"service_types" : "SELECT *, now() AS '_source_loaded_at' FROM service_types;",
|
||||
"transactions" : "SELECT *, now() AS '_source_loaded_at' FROM transactions;",
|
||||
"transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM `transaction_detail`;",
|
||||
"users" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, MD5(email) AS email_hash_id, type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM users;",
|
||||
"user_email_verifications" : "SELECT id, MD5(email) AS email_hash_id, is_complete, is_active, is_sent, created_at, updated_at, now() AS '_source_loaded_at' FROM user_email_verifications;",
|
||||
"wallets" : "SELECT id, owner_type, owner_id, CONCAT(SUBSTR(code,1,2),SUBSTR(code,5,1),REPEAT('',LOCATE('@',code)-3),SUBSTR(code,LOCATE('@',code)-1,3),REPEAT('',LENGTH(code)-LOCATE('.',REVERSE(code))-LOCATE('@',code)-1),SUBSTR(code,LENGTH(code)-LOCATE('.',REVERSE(code)))) AS code, currency_id, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM wallets;"
|
||||
"activity_log" : "SELECT * FROM activity_log;",
|
||||
"addresses" : "SELECT * FROM addresses;",
|
||||
"announcements" : "SELECT * FROM announcements;",
|
||||
"announcement_segment" : "SELECT * FROM announcement_segment;",
|
||||
"banks" : "SELECT * FROM banks;",
|
||||
"bank_logs" : "SELECT * FROM bank_logs;",
|
||||
"bookings" : "SELECT * FROM bookings;",
|
||||
"companies" : "SELECT * FROM companies;",
|
||||
"contacts" : "SELECT * FROM contacts;",
|
||||
"countries" : "SELECT * FROM countries;",
|
||||
"currencies" : "SELECT * FROM currencies;",
|
||||
"currency_logs" : "SELECT * FROM currency_logs;",
|
||||
"currency_rates" : "SELECT * FROM currency_rates;",
|
||||
"currency_rate_logs" : "SELECT * FROM currency_rate_logs;",
|
||||
"districts" : "SELECT * FROM districts;",
|
||||
"documents" : "SELECT * FROM documents;",
|
||||
"employees" : "SELECT * FROM employees;",
|
||||
"failed_jobs" : "SELECT * FROM failed_jobs;",
|
||||
"files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '<br>'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
|
||||
"groups" : "SELECT * FROM groups;",
|
||||
"group_transactions" : "SELECT * FROM group_transactions;",
|
||||
"jobs" : "SELECT * FROM jobs;",
|
||||
"migrations" : "SELECT * FROM migrations;",
|
||||
"model_has_roles" : "SELECT * FROM model_has_roles;",
|
||||
"notifications" : "SELECT * FROM notifications;",
|
||||
"password_resets" : "SELECT * FROM password_resets;",
|
||||
"permissions" : "SELECT * FROM permissions;",
|
||||
"roles" : "SELECT * FROM roles;",
|
||||
"role_has_permissions" : "SELECT * FROM role_has_permissions;",
|
||||
"segments" : "SELECT * FROM segments;",
|
||||
"segment_companies" : "SELECT * FROM segment_companies;",
|
||||
"segment_constants" : "SELECT * FROM segment_constants;",
|
||||
"service_types" : "SELECT * FROM service_types;",
|
||||
"states" : "SELECT * FROM states;",
|
||||
"transactions" : "SELECT * FROM transactions;",
|
||||
"transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', '<br>'),'|', '|'),'\\\\','\\') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', '<br>'),'|', '|'),'\\\\','\\') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;",
|
||||
"users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
|
||||
"user_email_verifications" : "SELECT * FROM user_email_verifications;",
|
||||
"wallets" : "SELECT * FROM wallets;"
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
{
|
||||
"activity_log" : "SELECT * FROM activity_log;",
|
||||
"addresses" : "SELECT * FROM addresses;",
|
||||
"announcements" : "SELECT * FROM announcements;",
|
||||
"announcement_segment" : "SELECT * FROM announcement_segment;",
|
||||
// "banks" : "SELECT * FROM banks;",
|
||||
// "bank_logs" : "SELECT * FROM bank_logs;",
|
||||
// "bookings" : "SELECT * FROM bookings;",
|
||||
"companies" : "SELECT * FROM companies;",
|
||||
--"company_connections" : "SELECT * FROM company_connections;",
|
||||
--"company_employees" : "SELECT * FROM company_employees;",
|
||||
--"company_modules" : "SELECT * FROM company_modules;",
|
||||
--"connection_segments" : "SELECT * FROM connection_segments;",
|
||||
"company_connections" : "SELECT * FROM company_connections;",
|
||||
"company_employees" : "SELECT * FROM company_employees;",
|
||||
"company_modules" : "SELECT * FROM company_modules;",
|
||||
"connection_segments" : "SELECT * FROM connection_segments;",
|
||||
"contacts" : "SELECT * FROM contacts;",
|
||||
--"containers" : "SELECT * FROM countries",
|
||||
--"container_packing_lists" : "SELECT * FROM container_packing_lists",
|
||||
"containers" : "SELECT * FROM containers",
|
||||
"container_packing_lists" : "SELECT * FROM container_packing_lists",
|
||||
"countries" : "SELECT * FROM countries;",
|
||||
"currencies" : "SELECT * FROM currencies;",
|
||||
// "currency_logs" : "SELECT * FROM currency_logs;",
|
||||
// "currency_rates" : "SELECT * FROM currency_rates;",
|
||||
// "currency_rate_logs" : "SELECT * FROM currency_rate_logs;",
|
||||
"districts" : "SELECT * FROM districts;",
|
||||
"documents" : "SELECT * FROM documents;",
|
||||
// "employees" : "SELECT * FROM employees;",
|
||||
"failed_jobs" : "SELECT * FROM failed_jobs;",
|
||||
"files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', ''),'|', ''),'\\\\','') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
|
||||
// "groups" : "SELECT * FROM groups;",
|
||||
// "group_transactions" : "SELECT * FROM group_transactions;",
|
||||
"jobs" : "SELECT * FROM jobs;",
|
||||
"files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '<br>'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
|
||||
"migrations" : "SELECT * FROM migrations;",
|
||||
// "model_has_roles" : "SELECT * FROM model_has_roles;",
|
||||
// "notifications" : "SELECT * FROM notifications;",
|
||||
--"orders" : "SELECT * FROM orders",
|
||||
--"order_roles" : "SELECT * FROM order_roles",
|
||||
--"packages" : "SELECT * FROM packages",
|
||||
--"packing_lists" : "SELECT * FROM packing_lists",
|
||||
"orders" : "SELECT * FROM orders",
|
||||
"order_roles" : "SELECT * FROM order_roles",
|
||||
"packages" : "SELECT * FROM packages",
|
||||
"packing_lists" : "SELECT * FROM packing_lists",
|
||||
"password_resets" : "SELECT * FROM password_resets;",
|
||||
// "permissions" : "SELECT * FROM permissions;",
|
||||
--"remarks" : "SELECT * FROM remarks",
|
||||
// "roles" : "SELECT * FROM roles;",
|
||||
// "role_has_permissions" : "SELECT * FROM role_has_permissions;",
|
||||
--"schedules" : "SELECT * FROM schedules",
|
||||
"remarks" : "SELECT id, owner_type, owner_id, commenter_id, REPLACE(REPLACE(REPLACE(content, '\n', '<br>'),'|', '|'),'\\\\','\\') AS 'content', deleted_at, created_at, updated_at FROM remarks",
|
||||
"schedules" : "SELECT * FROM schedules",
|
||||
"segments" : "SELECT * FROM segments;",
|
||||
// "segment_companies" : "SELECT * FROM segment_companies;",
|
||||
"segment_constants" : "SELECT * FROM segment_constants;",
|
||||
// "service_types" : "SELECT * FROM service_types;",
|
||||
"states" : "SELECT * FROM states;",
|
||||
--"steps" : "SELECT * FROM steps",
|
||||
"steps" : "SELECT * FROM steps",
|
||||
"transactions" : "SELECT * FROM transactions;",
|
||||
"transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;",
|
||||
--"transport_arrangements" : "SELECT * FROM transport_arrangements",
|
||||
"transaction_details" : "SELECT * FROM `transaction_details`;",
|
||||
"transport_arrangements" : "SELECT * FROM transport_arrangements",
|
||||
"users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
|
||||
"user_email_verifications" : "SELECT * FROM user_email_verifications;",
|
||||
// "wallets" : "SELECT * FROM wallets;"
|
||||
"user_email_verifications" : "SELECT * FROM user_email_verifications;"
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"identifier" : "0f8edfb8-1271-4e8c-bdfc-e33de84a0efe",
|
||||
"key" : "408094e63ab670c968b3f9a119387263d4e23721817513b47bd94700bf4349a9",
|
||||
"website_id" : "665dcd41-1edf-4451-8cb9-f1cf9ed35e15"
|
||||
}
|
||||
@@ -1,43 +1,43 @@
|
||||
{
|
||||
"activity_log" : "SELECT * FROM activity_log;",
|
||||
"addresses" : "SELECT * FROM addresses;",
|
||||
"activity_log" : "SELECT ID, LOG_NAME, DESCRIPTION, SUBJECT_ID, SUBJECT_TYPE, CAUSER_ID, CAUSER_TYPE, '**MASKING_PROPERTIES**' AS PROPERTIES, CREATED_AT, UPDATED_AT FROM activity_log;",
|
||||
"addresses" : "SELECT ID, COMPANY_ID, COUNTRY_ID, STATE_ID, DISTRICT_ID, POSTCODE, '**MASKING_STREET_ONE**'AS STREET_ONE, '**MASKING_STREET_TWO**'AS STREET_TWO, BILLING, DELETED_AT, CREATED_AT, UPDATED_AT FROM addresses;",
|
||||
"announcements" : "SELECT * FROM announcements;",
|
||||
"announcement_segment" : "SELECT * FROM announcement_segment;",
|
||||
"banks" : "SELECT * FROM banks;",
|
||||
"bank_logs" : "SELECT * FROM bank_logs;",
|
||||
"banks" : "SELECT ID, COMPANY_ID, '**MASKING_REFERENCE**' AS REFERENCE, BANK_NAME, MD5(CONCAT(HOLDER_NAME,LENGTH(HOLDER_NAME)*2022*2)) AS HOLDER_NAME, MD5(CONCAT(ACCOUNT_NO,LENGTH(ACCOUNT_NO)*2022*2)) AS ACCOUNT_NO, BANK_BRANCH, MD5(CONCAT(SWIFT,LENGTH(SWIFT)*2022*2)) AS SWIFT, MD5(CONCAT(SNAP,LENGTH(SNAP)*2022*2)) AS SNAP, TYPE, DEFAULT, STATUS, COUNTRY_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM banks;",
|
||||
"bank_logs" : "SELECT ID, BANK_ID, COMPANY_ID, '**MASKING_REFERENCE**' AS REFERENCE, BANK_NAME, MD5(CONCAT(HOLDER_NAME,LENGTH(HOLDER_NAME)*2022*2)) AS HOLDER_NAME, MD5(CONCAT(ACCOUNT_NO,LENGTH(ACCOUNT_NO)*2022*2)) AS ACCOUNT_NO, BANK_BRANCH, MD5(CONCAT(SWIFT,LENGTH(SWIFT)*2022*2)) AS SWIFT, MD5(CONCAT(SNAP,LENGTH(SNAP)*2022*2)) AS SNAP, TYPE, DEFAULT, STATUS, COUNTRY_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM bank_logs;",
|
||||
"bookings" : "SELECT * FROM bookings;",
|
||||
"companies" : "SELECT * FROM companies;",
|
||||
"contacts" : "SELECT * FROM contacts;",
|
||||
"companies" : "SELECT ID, '**MASKING_NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, DEBTOR, TYPE, BUSINESS_TYPE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM companies;",
|
||||
"contacts" : "SELECT ID, COMPANY_ID, COUNTRY_ID, '**MASKING_REFERENCE**' AS REFERENCE, MD5(CONCAT(PHONE,LENGTH(PHONE)*2022*2)) AS PHONE, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, MD5(CONCAT(WECHAT_ID,LENGTH(WECHAT_ID)*2022*2)) AS WECHAT_ID, DELETED_AT, CREATED_AT, UPDATED_AT FROM contacts;",
|
||||
"countries" : "SELECT * FROM countries;",
|
||||
"currencies" : "SELECT * FROM currencies;",
|
||||
"currency_logs" : "SELECT * FROM currency_logs;",
|
||||
"currency_rates" : "SELECT * FROM currency_rates;",
|
||||
"currency_rate_logs" : "SELECT * FROM currency_rate_logs;",
|
||||
"districts" : "SELECT * FROM districts;",
|
||||
"documents" : "SELECT * FROM documents;",
|
||||
"documents" : "SELECT ID, OWNER_TYPE, OWNER_ID, DOCUMENT_TYPE, '**MASKING_REFERENCE**' AS REFERENCE, STATUS, ISSUED_DATE, EXPIRED_DATE, APPROVER, APPROVAL_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM documents;",
|
||||
"employees" : "SELECT * FROM employees;",
|
||||
"failed_jobs" : "SELECT * FROM failed_jobs;",
|
||||
"files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '<br>'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
|
||||
"groups" : "SELECT * FROM groups;",
|
||||
"failed_jobs" : "SELECT ID, CONNECTION, QUEUE, '**MASKING_PAYLOAD' AS PAYLOAD, '**MASKING_EXCEPTION**' AS EXCEPTION, FAILED_AT FROM failed_jobs;",
|
||||
"files" : "SELECT id, document_id, '**MASKING_FILE**', file_type, deleted_at, created_at, updated_at FROM files;",
|
||||
"groups" : "SELECT ID, '**MASKING_REFERENCE**' AS REFERENCE, ISSUER, RECEIVER, AMOUNT, ORIGINAL_AMOUNT, CURRENCY_ID, ORIGINAL_CURRENCY_ID, CURRENCY_RATE, TAX, SERVICE_CHARGE, STATUS, CREATED_AT, UPDATED_AT FROM groups;",
|
||||
"group_transactions" : "SELECT * FROM group_transactions;",
|
||||
"jobs" : "SELECT * FROM jobs;",
|
||||
"jobs" : "SELECT ID, QUEUE, '**MASKING_PAYLOAD**' AS PAYLOAD, ATTEMPTS, RESERVED_AT, AVAILABLE_AT, CREATED_AT FROM jobs;",
|
||||
"migrations" : "SELECT * FROM migrations;",
|
||||
"model_has_roles" : "SELECT * FROM model_has_roles;",
|
||||
"notifications" : "SELECT * FROM notifications;",
|
||||
"password_resets" : "SELECT * FROM password_resets;",
|
||||
"password_resets" : "SELECT ID, USER_ID, '**MASKING_TOKEN**' AS TOKEN, IS_EXPIRED, IS_COMPLETE, CREATED_AT, UPDATED_AT FROM password_resets;",
|
||||
"permissions" : "SELECT * FROM permissions;",
|
||||
"roles" : "SELECT * FROM roles;",
|
||||
"role_has_permissions" : "SELECT * FROM role_has_permissions;",
|
||||
"segments" : "SELECT * FROM segments;",
|
||||
"segment_companies" : "SELECT * FROM segment_companies;",
|
||||
"segment_constants" : "SELECT * FROM segment_constants;",
|
||||
"segment_constants" : "SELECT ID, SEGMENT_ID, NAME, REFERENCE, '**MASKING_DETAIL**' AS DETAIL, DELETED_AT, CREATED_AT, UPDATED_AT FROM segment_constants;",
|
||||
"service_types" : "SELECT * FROM service_types;",
|
||||
"states" : "SELECT * FROM states;",
|
||||
"transactions" : "SELECT * FROM transactions;",
|
||||
"transactions" : "SELECT ID, OWNER_TYPE, OWNER_ID, TYPE, ISSUER, RECEIVER, RECIPIENT_BANK_ACCOUNT_ID, PAYMENT_METHOD, '**PAYMENT_REFERENCE**' AS PAYMENT_REFERENCE, MD5(CONCAT(BILL_NO,LENGTH(BILL_NO)*2022*2)) AS BILL_NO, AMOUNT, ORIGINAL_AMOUNT, CURRENCY_ID, ORIGINAL_CURRENCY_ID, CURRENCY_RATE, TAX, SERVICE_CHARGE, EXPIRES_ON, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM transactions;",
|
||||
"transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', '<br>'),'|', '|'),'\\\\','\\') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', '<br>'),'|', '|'),'\\\\','\\') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at FROM `transaction_detail`;",
|
||||
"users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
|
||||
"user_email_verifications" : "SELECT * FROM user_email_verifications;",
|
||||
"wallets" : "SELECT * FROM wallets;"
|
||||
"users" : "SELECT id, '**MASKING_NAME**' AS name, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
|
||||
"user_email_verifications" : "SELECT ID, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, '**MASKING_TOKEN**' AS TOKEN, IS_COMPLETE, IS_ACTIVE, IS_SENT, CREATED_AT, UPDATED_AT FROM user_email_verifications;",
|
||||
"wallets" : "SELECT ID, OWNER_TYPE, OWNER_ID, MD5(CONCAT(CODE,LENGTH(CODE)*2022*2)) AS CODE, CURRENCY_ID, AMOUNT, DELETED_AT, CREATED_AT, UPDATED_AT FROM wallets;"
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,37 +1,37 @@
|
||||
{
|
||||
"activity_log" : "SELECT * FROM activity_log;",
|
||||
"addresses" : "SELECT * FROM addresses;",
|
||||
"activity_log" : "SELECT ID, LOG_NAME, DESCRIPTION, SUBJECT_TYPE, SUBJECT_ID, CAUSER_TYPE, CAUSER_ID, '**MASKING_PROPERTIES**' AS PROPERTIES, CREATED_AT, UPDATED_AT FROM activity_log;",
|
||||
"addresses" : "SELECT ID, OWNER_TYPE, OWNER_ID, '**MASKING_REFERENCE**' AS REFERENCE, '**MASKING_STREET_ONE**' AS STREET_ONE, '**MASKING_STREET_TWO**'AS STREET_TWO, DISTRICT_ID, STATE_ID, COUNTRY_ID, POSTCODE, TYPE, DEFAULT, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM addresses;",
|
||||
"announcements" : "SELECT * FROM announcements;",
|
||||
"announcement_segment" : "SELECT * FROM announcement_segment;",
|
||||
"companies" : "SELECT * FROM companies;",
|
||||
"company_connections" : "SELECT * FROM company_connections;",
|
||||
"companies" : "SELECT ID, '**MASKING_NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, DEBTOR, TYPE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM companies;",
|
||||
"company_connections" : "SELECT ID, INVITER_ID, INVITEE_ID, '**MASKING INVITER_REFERENCE**' AS INVITER_REFERENCE, '**MASKING INVITEE_REFERENCE**' AS INVITEE_REFERENCE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM company_connections;",
|
||||
"company_employees" : "SELECT * FROM company_employees;",
|
||||
"company_modules" : "SELECT * FROM company_modules;",
|
||||
"company_modules" : "SELECT ID, COMPANY_ID, '**MASKING NAME**' AS NAME, '**MASKING_REFERENCE**' AS REFERENCE, TYPE, STATUS, MD5(CONCAT(UNITY_HASH_ID,LENGTH(UNITY_HASH_ID)*2022*2)) AS UNITY_HASH_ID, MD5(CONCAT(UNITY_SIGNATURE,LENGTH(UNITY_SIGNATURE)*2022*2)) AS UNITY_SIGNATURE, DELETED_AT, CREATED_AT, UPDATED_AT FROM company_modules;",
|
||||
"connection_segments" : "SELECT * FROM connection_segments;",
|
||||
"contacts" : "SELECT * FROM contacts;",
|
||||
"contacts" : "SELECT ID, OWNER_TYPE, OWNER_ID, '**MASKING_REFERENCE**' AS REFERENCE, MD5(CONCAT(PHONE,LENGTH(PHONE)*2022*2)) AS PHONE, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, MD5(CONCAT(WECHAT_ID,LENGTH(WECHAT_ID)*2022*2)) AS WECHAT_ID, DEFAULT, DELETED_AT, CREATED_AT, UPDATED_AT FROM contacts;",
|
||||
"containers" : "SELECT * FROM containers",
|
||||
"container_packing_lists" : "SELECT * FROM container_packing_lists",
|
||||
"countries" : "SELECT * FROM countries;",
|
||||
"currencies" : "SELECT * FROM currencies;",
|
||||
"districts" : "SELECT * FROM districts;",
|
||||
"documents" : "SELECT * FROM documents;",
|
||||
"failed_jobs" : "SELECT * FROM failed_jobs;",
|
||||
"files" : "SELECT id, document_id, REPLACE(REPLACE(REPLACE(file, '\n', '<br>'),'|', '|'),'\\\\','\\') AS 'file', file_type, deleted_at, created_at, updated_at FROM files;",
|
||||
"documents" : "SELECT ID, OWNER_TYPE, OWNER_ID, DOCUMENT_TYPE, '**MASKING_REFERENCE**' AS REFERENCE, STATUS, ISSUED_DATE, EXPIRED_DATE, APPROVER, APPROVAL_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM documents;",
|
||||
"failed_jobs" : "SELECT ID, UUID, CONNECTION, QUEUE, '**MASKING_PAYLOAD' AS PAYLOAD, '**MASKING_EXCEPTION**' AS EXCEPTION, FAILED_AT FROM failed_jobs;",
|
||||
"files" : "SELECT id, document_id, '**MASKING FILE**' AS FILE, file_type, deleted_at, created_at, updated_at FROM files;",
|
||||
"migrations" : "SELECT * FROM migrations;",
|
||||
"orders" : "SELECT * FROM orders",
|
||||
"order_roles" : "SELECT * FROM order_roles",
|
||||
"order_roles" : "SELECT ID, ORDER_ID, COMPANY_MODULE_ID, ROLE_ID, MD5(CONCAT(ENTITY_HASH_ID,LENGTH(ENTITY_HASH_ID)*2022*2)) AS ENTITY_HASH_ID. MD5(CONCAT(ENTITY_SIGNATURE,LENGTH(ENTITY_SIGNATURE)*2022*2)) AS ENTITY_SIGNATURE, DELETED_AT, CREATED_AT, UPDATED_AT FROM order_roles",
|
||||
"packages" : "SELECT * FROM packages",
|
||||
"packing_lists" : "SELECT * FROM packing_lists",
|
||||
"password_resets" : "SELECT * FROM password_resets;",
|
||||
"remarks" : "SELECT id, owner_type, owner_id, commenter_id, REPLACE(REPLACE(REPLACE(content, '\n', '<br>'),'|', '|'),'\\\\','\\') AS 'content', deleted_at, created_at, updated_at FROM remarks",
|
||||
"password_resets" : "SELECT ID, USER_ID, '**MASKING_TOKEN**' AS TOKEN, IS_EXPIRED, IS_COMPLETE, CREATED_AT, UPDATED_AT FROM password_resets;",
|
||||
"remarks" : "SELECT id, owner_type, owner_id, commenter_id, '**MASKING CONTENT**' AS CONTENT, deleted_at, created_at, updated_at FROM remarks",
|
||||
"schedules" : "SELECT * FROM schedules",
|
||||
"segments" : "SELECT * FROM segments;",
|
||||
"segment_constants" : "SELECT * FROM segment_constants;",
|
||||
"states" : "SELECT * FROM states;",
|
||||
"steps" : "SELECT * FROM steps",
|
||||
"steps" : "SELECT ID, OWNER_TYPE, OWNER_ID, APPOINTEE_ID, MD5(CONCAT(OBLIGATION_HASH_ID,LENGTH(OBLIGATION_HASH_ID)*2022*2)) AS OBLIGATION_HASH_ID, REFERENCE, PRIMARY, SEQUENCE, STATUS, COMPLETE_DATE, DELETED_AT, CREATED_AT, UPDATED_AT FROM steps",
|
||||
"transactions" : "SELECT * FROM transactions;",
|
||||
"transaction_details" : "SELECT * FROM `transaction_details`;",
|
||||
"transport_arrangements" : "SELECT * FROM transport_arrangements",
|
||||
"users" : "SELECT id, name, email, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
|
||||
"user_email_verifications" : "SELECT * FROM user_email_verifications;"
|
||||
"transport_arrangements" : "SELECT ID, OWNER_TYPE, OWNER_ID, TYPE, COURIER, LEFT(TRACKING_NUMBER,6) AS TRACKING_NUMBER, DISPATCH_DATE, DROP_DATE, STATUS, DELETED_AT, CREATED_AT, UPDATED_AT FROM transport_arrangements",
|
||||
"users" : "SELECT id, '**MASKING NAME**' AS NAME, MD5(CONCAT(EMAIL,LENGTH(EMAIL)*2022*2)) AS EMAIL, type, status, remember_token, active_at, deleted_at, created_at, updated_at FROM users;",
|
||||
"user_email_verifications" : "SELECT ID, MD5(CONCAT(email,LENGTH(email)*2022*2)) AS email, '**MASKING_TOKEN**' AS TOKEN, IS_COMPLETE, IS_ACTIVE, IS_SENT, CREATED_AT, UPDATED_AT FROM user_email_verifications;"
|
||||
}
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,61 @@
|
||||
from crisp_api import Crisp
|
||||
import general_function
|
||||
import pandas as pd
|
||||
|
||||
def connect_crisp(config):
|
||||
try:
|
||||
client = Crisp()
|
||||
client.set_tier("plugin")
|
||||
client.authenticate(config["identifier"], config["key"])
|
||||
return client
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("[FUNCTION_ERROR]-connect_crisp")
|
||||
return False
|
||||
|
||||
def get_operators_csv(website_id, client, path, filename):
|
||||
try:
|
||||
output_location = path + filename
|
||||
operators_list_json = client.website.list_website_operators(website_id)
|
||||
operators_list_df = general_function.json_normalize_to_df(operators_list_json)
|
||||
operators_list_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
print("[Download Complete]",filename)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("[FUNCTION_ERROR]-get_operators_csv")
|
||||
return False
|
||||
|
||||
def get_peoples_csv(website_id, client, path, filename):
|
||||
try:
|
||||
output_location = path + filename
|
||||
peoples_list_df = pd.DataFrame()
|
||||
peoples_data_df = pd.DataFrame()
|
||||
loop_flag = True
|
||||
page_number = 1
|
||||
|
||||
while loop_flag == True:
|
||||
peoples_list_json = client.website.list_people_profiles(website_id, page_number)
|
||||
temp_df = general_function.json_normalize_to_df(peoples_list_json)
|
||||
peoples_list_df = pd.concat([temp_df,peoples_list_df], ignore_index=True)
|
||||
if not peoples_list_json:
|
||||
loop_flag = False
|
||||
else:
|
||||
page_number += 1
|
||||
|
||||
for people_id in peoples_list_df.people_id:
|
||||
print(people_id)
|
||||
people_data_json = client.website.get_people_data(website_id, people_id)
|
||||
temp_df = general_function.json_normalize_to_df(people_data_json)
|
||||
temp_df['people_id'] = people_id
|
||||
peoples_data_df = pd.concat([temp_df,peoples_data_df], ignore_index=True)
|
||||
|
||||
peoples_list_df = peoples_list_df.merge(peoples_data_df, on='people_id', how='left')
|
||||
|
||||
peoples_list_df.to_csv(output_location, sep=',', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
peoples_data_df.to_csv('test.csv', sep=',', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
print("[Download Complete]",filename)
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("[FUNCTION_ERROR]-get_peoples_csv")
|
||||
return False
|
||||
@@ -0,0 +1,20 @@
|
||||
type|details.user_id|details.email|details.avatar|details.first_name|details.last_name|details.role|details.title|details.availability|details.has_token
|
||||
operator|15f0ec35-8a0c-457f-83ad-68e04f3b6624|taypeisongtps@gmail.com|https://storage.crisp.chat/users/avatar/operator/5e71c78373d9ac00/potrait-photo_gsa6gd.jpg|Pei Song|Tay|member|Customer Service|away|False
|
||||
operator|293073d9-e65f-40e2-a1ca-54fb9a2e4ff7|ykheng2bb@gmail.com|https://storage.crisp.chat/users/avatar/operator/4a7ce09c05d7b400/whatsapp-image-2022-04-13-at-2_19jwcch.jpeg|YAP|KAH HENG|owner|Customer Service|online|False
|
||||
operator|294351f3-d501-41dd-a095-2c512dd2ff4f|ciefworldwide.acc1@gmail.com||Haizat|Zaidi|owner||away|False
|
||||
operator|2a0b5503-97ff-4875-918d-be8185bc58a1|h2009413007@hcu.edu.my|https://storage.crisp.chat/users/avatar/operator/6bba6be706925c00/screenshot-2022-09-30-150004_mst2g.png|Tok|soon wee|member|Consultant|online|False
|
||||
operator|413c155c-3f3b-4859-bc18-dddb17dfd3ac|diliniwr10@gmail.com|https://storage.crisp.chat/users/avatar/operator/ed6e6534f8f01000/dili_wntzti.png|Dilini|WR|member||away|False
|
||||
operator|6f231d14-b3c0-47b5-bd02-617579cd2bcb|kangjun8331@gmail.com|https://storage.crisp.chat/users/avatar/operator/9ccd7bf346431800/img1413_d7cf1m.jpg|Kang|Jun|member|Consultant|away|False
|
||||
operator|80708bf4-9c08-4b8d-a37b-3a2f3c9570f1|jhunyanwong@gmail.com|https://storage.crisp.chat/users/avatar/operator/b20140b4cfccf800/214025936361797548903332433376_1r1o1en.jpg|JY|Wong|owner||away|False
|
||||
operator|83fb3451-f17a-4876-b317-244c30b27c8b|shafiqa_sukeri@cief-malaysia.com|https://storage.crisp.chat/users/avatar/operator/6ba11d171dddb800/photo2022-07-0811-15-15_9ulegi.jpg|FAIN|BACKEND|owner|Customer Service|online|False
|
||||
operator|99a1c001-60ac-4f71-91e1-33e8c2205525|enxin1227@gmail.com|https://storage.crisp.chat/users/upload/operator/bb7c7077d72bc800/21064570005_zf9jf6.jpg|Tiffany|Ooi|member||online|False
|
||||
operator|9df5d57c-8eb6-4e43-a8c5-cc6282564de9|ciefworldwide@gmail.com||PM|Wong|owner|CEO|away|False
|
||||
operator|a8079229-94c0-4a33-a027-bcbaed17a1c0|uldvstar@gmail.com|https://storage.crisp.chat/users/avatar/operator/8f0372fcb5eb0000/download-1_17y48zb.png|Omair|Saleh|owner||away|False
|
||||
operator|d8e200bc-709b-4b80-a706-bab8ec612e0b|infociefworldwide@gmail.com||CIEF|Marketing|owner|Marketing|away|False
|
||||
operator|d9ef0024-60c9-4137-919d-1d25bc2fab0e|operationteam@cief-malaysia.com||Operation|Team|member||away|False
|
||||
operator|db57d7a6-9421-43b9-9614-0867db22739a|ziwen38@gmail.com|https://storage.crisp.chat/users/avatar/operator/eed9c8d96123e00/resume_1x03t8n.jpeg|Zi Wen|Lee|member||online|False
|
||||
operator|dd8d2dcb-07a4-4343-b5bd-93038d08bd54|lingzunsien1211@gmail.com|https://storage.crisp.chat/users/avatar/operator/67b2c5d990ba9800/resume-pic_1q3r11v.jpg|Zun Sien|Ling|member||online|False
|
||||
operator|e486c123-0501-43e7-aa00-0231fb589c2e|potter2186@gmail.com||Harry|Wong|owner|Manager|away|False
|
||||
operator|f43c636e-c023-4169-a555-be63a1f34a12|sookni98@gmail.com|https://storage.crisp.chat/users/avatar/operator/db5dc231ebfc1000/ksn-5_unmi7u.jpeg|Sook Ni|Kho|member||online|False
|
||||
operator|fcbaf32c-29dc-4a24-af7d-77ceeb8616b9|yamzhenglim0427@gmail.com||ZhengLim|Yam|owner||online|False
|
||||
sandbox|631dc1f5-f672-41a6-b32a-0dc05339c97f|yamzhenglim0427@gmail.com||ZhengLim|Yam||||
|
||||
|
Binary file not shown.
@@ -1,6 +1,7 @@
|
||||
import json
|
||||
from operator import concat
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
def read_json_file(path, name):
|
||||
try:
|
||||
@@ -25,3 +26,13 @@ def delete_csv_in_path(path):
|
||||
def change_to_current_directory():
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
os.chdir(script_dir)
|
||||
|
||||
|
||||
def json_normalize_to_df(json):
|
||||
try:
|
||||
df = pd.json_normalize(json)
|
||||
return df
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("[FUNCTION_ERROR]-json_normalize_to_df")
|
||||
return False
|
||||
@@ -5,6 +5,8 @@ import exchange.extract.exchange_extract_function as exchange_extract_func
|
||||
import exchange.load.exchange_load_function as exchange_load_func
|
||||
import shipping.extract.shipping_extract_function as shipping_extract_func
|
||||
import shipping.load.shipping_load_function as shipping_load_func
|
||||
import crisp.extract.crisp_extract_function as crisp_extract_func
|
||||
|
||||
|
||||
def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
|
||||
#Config file name
|
||||
@@ -40,8 +42,6 @@ def exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH):
|
||||
file_path = os.path.join(dirpath,file_name)
|
||||
exchange_load_func.upload_to_bucket(ex_storage_client, file_name, file_path)
|
||||
|
||||
|
||||
|
||||
def shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH):
|
||||
#Config file name
|
||||
SHIPPING_MYSQL_CONFIG_FILE = "SHIPPING_CONFIG.json"
|
||||
@@ -59,7 +59,6 @@ def shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH):
|
||||
#Run EX_SQL_DICT to get all data from mysql in csv format
|
||||
shipping_extract_func.mysql_dict_sql_to_csv(SHIPPING_SQL_DICT, SHIPPING_CSV_TEMP_STORAGE_PATH, SHIPPING_MYSQLDB, SHIPPING_MYSQLCURSOR)
|
||||
|
||||
|
||||
def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
|
||||
#Config file name
|
||||
CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
|
||||
@@ -77,8 +76,25 @@ def shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH):
|
||||
file_path = os.path.join(dirpath,file_name)
|
||||
shipping_load_func.upload_to_bucket(shipping_storage_client, file_name, file_path)
|
||||
|
||||
def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
|
||||
#Config file name
|
||||
CRISP_API_CONFIG_FILE = "CRISP_CONFIG.json"
|
||||
CRISP_CSV_TEMP_STORAGE_PATH = CRISP_CSV_TEMP_STORAGE_PATH
|
||||
|
||||
#Read config file
|
||||
CRISP_API_CONFIG = general_function.read_json_file("./config/",CRISP_API_CONFIG_FILE)
|
||||
|
||||
#Get website_id
|
||||
CRISP_WEBSITE_ID = CRISP_API_CONFIG['website_id']
|
||||
|
||||
#Connect to CRISP
|
||||
CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG)
|
||||
|
||||
#Get operators csv
|
||||
crisp_extract_func.get_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_operators.csv')
|
||||
|
||||
# #Get users csv
|
||||
crisp_extract_func.get_peoples_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_peoples.csv')
|
||||
|
||||
def main():
|
||||
#Make sure cron in the file directory
|
||||
@@ -104,6 +120,12 @@ def main():
|
||||
general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
|
||||
# #Make sure cron in the file directory
|
||||
# general_function.change_to_current_directory()
|
||||
|
||||
# #Extract Load Exchange Data
|
||||
# CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
|
||||
# crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
mysql-connector-python==8.0.29
|
||||
google-cloud-storage==2.5.0
|
||||
crisp-api==1.1.13
|
||||
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user