cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Debug EL

Browse Source
This commit is contained in:
Yam ZhengLim
2022-10-29 01:31:08 +08:00
parent 5d30c6c505
commit d4d17efa3d
3 changed files with 142 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2 -1
View File
@@ -1,4 +1,5 @@
testing_sql.txt
internal_masking_doc/
config/local*.json
testing.sql
testing.sql
edmond.sql
config/EXCHANGE_EXTRACT_SQL.json
+1 -1
View File
@@ -34,7 +34,7 @@
"service_types" : "SELECT *, now() AS '_source_loaded_at' FROM service_types;",
"states" : "SELECT *, now() AS '_source_loaded_at' FROM states;",
"transactions" : "SELECT *, now() AS '_source_loaded_at' FROM transactions;",
"transaction_detail" : "SELECT *, now() AS '_source_loaded_at' FROM transaction_detail;",
"transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM `transaction_detail`;",
"users" : "SELECT *, now() AS '_source_loaded_at' FROM users;",
"user_email_verifications" : "SELECT *, now() AS '_source_loaded_at' FROM user_email_verifications;",
"wallets" : "SELECT *, now() AS '_source_loaded_at' FROM wallets;"
edmond.sql
+139 -31
View File
@@ -1,36 +1,144 @@
use database RAW;
use schema EDMOND_DEV;
use warehouse CIEF_DW_DEV;
USE WAREHOUSE DEV_CIEF_SNOWFLAKE_WH_XS;
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE role accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."BOOKINGS";
copy into "RAW"."EXCHANGE_MYSQL"."BOOKINGS"
from @exchange_gcs_stage
pattern=".*exchange_production_bookings.csv";
SELECT
bookings.id AS booking_id,
bookings.status AS booking_status,
bookings.company_id AS booking_company_id,
bookings.service_id AS booking_service_id,
bookings.fix_currency_id AS booking_fix_currency_id,
bookings.fix_amount AS booking_fix_booking_amount,
dim_currency_rate_logs.CURRENCY_RATE_LOGS_SELLING_RATE AS booking_base_to_quote_convertion_rate,
bookings.conversion_currency_id AS booking_base_currency_id,
IF (bookings.fix-currency-id == conversion(base)-curr-id THEN fix-amount ELSE fix-amount/CURRENCY_RATE_LOGS_SELLING_RATE)
-- bookings.idk AS booking_base_booking_amount
bookings.convertible_currency_id AS booking_quote_currency_id,
IF (fix-currency-id == convertible(quote)-curr-id THEN fix-amount ELSE fix-amount*CURRENCY_RATE_LOGS_SELLING_RATE) AS FSFSDFDSFDSF,
-- bookings.idk AS booking_quote_booking_amount
++ IF(base-currency = MYR THEN base -- bookings_amount_MYR
bookings.created_at, AS booking_created_at,
-- bookings.idk AS booking_expired_at
-- bookings.idk AS booking_is_expired
-- bookings.idk AS booking_approved_at
-- bookings.idk AS booking_is_approved
-- bookings.idk AS booking_completed_at
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."COMPANIES";
copy into "RAW"."EXCHANGE_MYSQL"."COMPANIES"
from @exchange_gcs_stage
pattern=".*exchange_production_companies.csv";
FROM RAW.EXCHANGE_MYSQL.BOOKINGS as bookings
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."CONTACTS";
copy into "RAW"."EXCHANGE_MYSQL"."CONTACTS"
from @exchange_gcs_stage
pattern=".*exchange_production_contacts.csv";
LEFT JOIN RAW.DBT_ZYAM_EXCHANGE_WAREHOUSE.DIM_CURRENCY_RATE_LOGS__EXCHANGE as dim_currency_rate_logs
ON (bookings.created_at > dim_currency_rate_logs .CURRENCY_RATE_LOGS_CREATED_DATE_FROM)
AND (bookings.created_at <= dim_currency_rate_logs .CURRENCY_RATE_LOGS_CREATED_DATE_TO)
AND (bookings.service_id = dim_currency_rate_logs .CURRENCY_RATE_LOGS_SERVICE_ID)
limit 1
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."COUNTRIES";
copy into "RAW"."EXCHANGE_MYSQL"."COUNTRIES"
from @exchange_gcs_stage
pattern=".*exchange_production_countries.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."CURRENCIES";
copy into "RAW"."EXCHANGE_MYSQL"."CURRENCIES"
from @exchange_gcs_stage
pattern=".*exchange_production_currencies.csv";
IFF(bookings.fix_currency_id = bookings.base_currency_id, bookings.fix_amount, bookings.fix_amount * dim_currency_rate_logs.CURRENCY_RATE_LOGS_SELLING_RATE)
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."CURRENCY_RATES";
copy into "RAW"."EXCHANGE_MYSQL"."CURRENCY_RATES"
from @exchange_gcs_stage
pattern=".*exchange_production_currency_rates.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."CURRENCY_RATE_LOGS";
copy into "RAW"."EXCHANGE_MYSQL"."CURRENCY_RATE_LOGS"
from @exchange_gcs_stage
pattern=".*exchange_production_currency_rate_logs.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."EMPLOYEES";
copy into "RAW"."EXCHANGE_MYSQL"."EMPLOYEES"
from @exchange_gcs_stage
pattern=".*exchange_production_employees.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."GROUPS";
copy into "RAW"."EXCHANGE_MYSQL"."GROUPS"
from @exchange_gcs_stage
pattern=".*exchange_production_groups.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."GROUP_TRANSACTIONS";
copy into "RAW"."EXCHANGE_MYSQL"."GROUP_TRANSACTIONS"
from @exchange_gcs_stage
pattern=".*exchange_production_group_transactions.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."SEGMENTS";
copy into "RAW"."EXCHANGE_MYSQL"."SEGMENTS"
from @exchange_gcs_stage
pattern=".*exchange_production_segments.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."SEGMENT_COMPANIES";
copy into "RAW"."EXCHANGE_MYSQL"."SEGMENT_COMPANIES"
from @exchange_gcs_stage
pattern=".*exchange_production_segment_companies.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."SERVICE_TYPES";
copy into "RAW"."EXCHANGE_MYSQL"."SERVICE_TYPES"
from @exchange_gcs_stage
pattern=".*exchange_production_service_types.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."TRANSACTIONS";
copy into "RAW"."EXCHANGE_MYSQL"."TRANSACTIONS"
from @exchange_gcs_stage
pattern=".*exchange_production_transactions.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."TRANSACTION_DETAIL";
copy into "RAW"."EXCHANGE_MYSQL"."TRANSACTION_DETAIL"
from @exchange_gcs_stage
pattern=".*exchange_production_transaction_detail.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."USERS";
copy into "RAW"."EXCHANGE_MYSQL"."USERS"
from @exchange_gcs_stage
pattern=".*exchange_production_users.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."USER_EMAIL_VERIFICATIONS";
copy into "RAW"."EXCHANGE_MYSQL"."USER_EMAIL_VERIFICATIONS"
from @exchange_gcs_stage
pattern=".*exchange_production_user_email_verifications.csv";
USE DATABASE DEV_CIEF_RAW_DB;
USE SCHEMA EXCHANGE_MYSQL;
USE ROLE accountadmin;
delete from "RAW"."EXCHANGE_MYSQL"."WALLETS";
copy into "RAW"."EXCHANGE_MYSQL"."WALLETS"
from @exchange_gcs_stage
pattern=".*exchange_production_wallets.csv";