From cb08e5b561de2a1bd7a157782944d1ac0444c568 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Fri, 28 Oct 2022 15:39:09 +0800 Subject: [PATCH] Update fully ELT --- EXCHANGE_EXTRACT_SQL_OLD.json | 22 +++++++++++++++++ config/EXCHANGE_EXTRACT_SQL.json | 41 ++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 10 deletions(-) create mode 100644 EXCHANGE_EXTRACT_SQL_OLD.json diff --git a/EXCHANGE_EXTRACT_SQL_OLD.json b/EXCHANGE_EXTRACT_SQL_OLD.json new file mode 100644 index 0000000..4c6c9a6 --- /dev/null +++ b/EXCHANGE_EXTRACT_SQL_OLD.json @@ -0,0 +1,22 @@ +{ + "bookings" : "SELECT id, company_id, marking, service_id, bank_id, fix_amount, fix_currency_id, convertible_currency_id, conversion_currency_id, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM bookings;", + "companies" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, reference, debtor, type, business_type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM companies;", + "contacts" : "SELECT id, company_id, country_id, CONCAT(SUBSTR(reference,1,2),SUBSTR(reference,5,1),REPEAT('',LOCATE('@',reference)-3),SUBSTR(reference,LOCATE('@',reference)-1,3),REPEAT('',LENGTH(reference)-LOCATE('.',REVERSE(reference))-LOCATE('@',reference)-1),SUBSTR(reference,LENGTH(reference)-LOCATE('.',REVERSE(reference)))) AS reference, REPLACE(phone,RIGHT(phone,round(length(phone)/2,0)+1),REPEAT('*', round(length(phone)/2+0.5,0))) as phone, CONCAT(SUBSTR(email,1,2),SUBSTR(email,5,1),REPEAT('',LOCATE('@',email)-3),SUBSTR(email,LOCATE('@',email)-1,3),REPEAT('',LENGTH(email)-LOCATE('.',REVERSE(email))-LOCATE('@',email)-1),SUBSTR(email,LENGTH(email)-LOCATE('.',REVERSE(email)))) AS email, CONCAT(SUBSTR(wechat_id,1,2),SUBSTR(wechat_id,5,1),REPEAT('',LOCATE('@',wechat_id)-3),SUBSTR(wechat_id,LOCATE('@',wechat_id)-1,3),REPEAT('',LENGTH(wechat_id)-LOCATE('.',REVERSE(wechat_id))-LOCATE('@',wechat_id)-1),SUBSTR(wechat_id,LENGTH(wechat_id)-LOCATE('.',REVERSE(wechat_id)))) AS wechat_id, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM contacts;", + "countries" : "SELECT *, now() AS '_source_loaded_at' FROM countries;", + "currencies" : "SELECT *, now() AS '_source_loaded_at' FROM currencies;", + "currency_rates" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rates;", + "currency_rate_logs" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rate_logs;", + "employees" : "SELECT *, now() AS '_source_loaded_at' FROM employees;", + "groups" : "SELECT *, now() AS '_source_loaded_at' FROM groups;", + "group_transactions" : "SELECT *, now() AS '_source_loaded_at' FROM group_transactions;", + "segments" : "SELECT *, now() AS '_source_loaded_at' FROM segments;", + "segment_companies" : "SELECT *, now() AS '_source_loaded_at' FROM segment_companies;", + "service_types" : "SELECT *, now() AS '_source_loaded_at' FROM service_types;", + "transactions" : "SELECT *, now() AS '_source_loaded_at' FROM transactions;", + "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM `transaction_detail`;", + "users" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, MD5(email) AS email_hash_id, type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM users;", + "user_email_verifications" : "SELECT id, MD5(email) AS email_hash_id, is_complete, is_active, is_sent, created_at, updated_at, now() AS '_source_loaded_at' FROM user_email_verifications;", + "wallets" : "SELECT id, owner_type, owner_id, CONCAT(SUBSTR(code,1,2),SUBSTR(code,5,1),REPEAT('',LOCATE('@',code)-3),SUBSTR(code,LOCATE('@',code)-1,3),REPEAT('',LENGTH(code)-LOCATE('.',REVERSE(code))-LOCATE('@',code)-1),SUBSTR(code,LENGTH(code)-LOCATE('.',REVERSE(code)))) AS code, currency_id, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM wallets;" +} + + diff --git a/config/EXCHANGE_EXTRACT_SQL.json b/config/EXCHANGE_EXTRACT_SQL.json index 4c6c9a6..acaccd1 100644 --- a/config/EXCHANGE_EXTRACT_SQL.json +++ b/config/EXCHANGE_EXTRACT_SQL.json @@ -1,22 +1,43 @@ { - "bookings" : "SELECT id, company_id, marking, service_id, bank_id, fix_amount, fix_currency_id, convertible_currency_id, conversion_currency_id, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM bookings;", - "companies" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, reference, debtor, type, business_type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM companies;", - "contacts" : "SELECT id, company_id, country_id, CONCAT(SUBSTR(reference,1,2),SUBSTR(reference,5,1),REPEAT('',LOCATE('@',reference)-3),SUBSTR(reference,LOCATE('@',reference)-1,3),REPEAT('',LENGTH(reference)-LOCATE('.',REVERSE(reference))-LOCATE('@',reference)-1),SUBSTR(reference,LENGTH(reference)-LOCATE('.',REVERSE(reference)))) AS reference, REPLACE(phone,RIGHT(phone,round(length(phone)/2,0)+1),REPEAT('*', round(length(phone)/2+0.5,0))) as phone, CONCAT(SUBSTR(email,1,2),SUBSTR(email,5,1),REPEAT('',LOCATE('@',email)-3),SUBSTR(email,LOCATE('@',email)-1,3),REPEAT('',LENGTH(email)-LOCATE('.',REVERSE(email))-LOCATE('@',email)-1),SUBSTR(email,LENGTH(email)-LOCATE('.',REVERSE(email)))) AS email, CONCAT(SUBSTR(wechat_id,1,2),SUBSTR(wechat_id,5,1),REPEAT('',LOCATE('@',wechat_id)-3),SUBSTR(wechat_id,LOCATE('@',wechat_id)-1,3),REPEAT('',LENGTH(wechat_id)-LOCATE('.',REVERSE(wechat_id))-LOCATE('@',wechat_id)-1),SUBSTR(wechat_id,LENGTH(wechat_id)-LOCATE('.',REVERSE(wechat_id)))) AS wechat_id, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM contacts;", + "activity_log" : "SELECT *, now() AS '_source_loaded_at' FROM activity_log;", + "addresses" : "SELECT *, now() AS '_source_loaded_at' FROM addresses;", + "announcements" : "SELECT *, now() AS '_source_loaded_at' FROM announcements;", + "announcement_segment" : "SELECT *, now() AS '_source_loaded_at' FROM announcement_segment;", + "banks" : "SELECT *, now() AS '_source_loaded_at' FROM banks;", + "bank_logs" : "SELECT *, now() AS '_source_loaded_at' FROM bank_logs;", + "bookings" : "SELECT *, now() AS '_source_loaded_at' FROM bookings;", + "companies" : "SELECT *, now() AS '_source_loaded_at' FROM companies;", + "contacts" : "SELECT *, now() AS '_source_loaded_at' FROM contacts;", "countries" : "SELECT *, now() AS '_source_loaded_at' FROM countries;", - "currencies" : "SELECT *, now() AS '_source_loaded_at' FROM currencies;", + "currencies" : "SELECT *, now() AS '_source_loaded_at' FROM currencies;", + "currency_logs" : "SELECT *, now() AS '_source_loaded_at' FROM currency_logs;", "currency_rates" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rates;", - "currency_rate_logs" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rate_logs;", + "currency_rate_logs" : "SELECT *, now() AS '_source_loaded_at' FROM currency_rate_logs;", + "districts" : "SELECT *, now() AS '_source_loaded_at' FROM districts;", + "documents" : "SELECT *, now() AS '_source_loaded_at' FROM documents;", "employees" : "SELECT *, now() AS '_source_loaded_at' FROM employees;", + "failed_jobs" : "SELECT *, now() AS '_source_loaded_at' FROM failed_jobs;", + "files" : "SELECT *, now() AS '_source_loaded_at' FROM files;", "groups" : "SELECT *, now() AS '_source_loaded_at' FROM groups;", - "group_transactions" : "SELECT *, now() AS '_source_loaded_at' FROM group_transactions;", + "group_transactions" : "SELECT *, now() AS '_source_loaded_at' FROM group_transactions;", + "jobs" : "SELECT *, now() AS '_source_loaded_at' FROM jobs;", + "migrations" : "SELECT *, now() AS '_source_loaded_at' FROM migrations;", + "model_has_roles" : "SELECT *, now() AS '_source_loaded_at' FROM model_has_roles;", + "notifications" : "SELECT *, now() AS '_source_loaded_at' FROM notifications;", + "password_resets" : "SELECT *, now() AS '_source_loaded_at' FROM password_resets;", + "permissions" : "SELECT *, now() AS '_source_loaded_at' FROM permissions;", + "roles" : "SELECT *, now() AS '_source_loaded_at' FROM roles;", + "role_has_permissions" : "SELECT *, now() AS '_source_loaded_at' FROM role_has_permissions;", "segments" : "SELECT *, now() AS '_source_loaded_at' FROM segments;", "segment_companies" : "SELECT *, now() AS '_source_loaded_at' FROM segment_companies;", + "segment_constants" : "SELECT *, now() AS '_source_loaded_at' FROM segment_constants;", "service_types" : "SELECT *, now() AS '_source_loaded_at' FROM service_types;", + "states" : "SELECT *, now() AS '_source_loaded_at' FROM states;", "transactions" : "SELECT *, now() AS '_source_loaded_at' FROM transactions;", - "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM `transaction_detail`;", - "users" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, MD5(email) AS email_hash_id, type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM users;", - "user_email_verifications" : "SELECT id, MD5(email) AS email_hash_id, is_complete, is_active, is_sent, created_at, updated_at, now() AS '_source_loaded_at' FROM user_email_verifications;", - "wallets" : "SELECT id, owner_type, owner_id, CONCAT(SUBSTR(code,1,2),SUBSTR(code,5,1),REPEAT('',LOCATE('@',code)-3),SUBSTR(code,LOCATE('@',code)-1,3),REPEAT('',LENGTH(code)-LOCATE('.',REVERSE(code))-LOCATE('@',code)-1),SUBSTR(code,LENGTH(code)-LOCATE('.',REVERSE(code)))) AS code, currency_id, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM wallets;" + "transaction_detail" : "SELECT *, now() AS '_source_loaded_at' FROM transaction_detail;", + "users" : "SELECT *, now() AS '_source_loaded_at' FROM users;", + "user_email_verifications" : "SELECT *, now() AS '_source_loaded_at' FROM user_email_verifications;", + "wallets" : "SELECT *, now() AS '_source_loaded_at' FROM wallets;" }