From d4d17efa3d470afd50d94ca5946737d24bc7a392 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Sat, 29 Oct 2022 01:31:08 +0800 Subject: [PATCH] Debug EL --- .gitignore | 3 +- config/EXCHANGE_EXTRACT_SQL.json | 2 +- edmond.sql | 170 +++++++++++++++++++++++++------ 3 files changed, 142 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 7aa8e17..7bbc2e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ testing_sql.txt internal_masking_doc/ config/local*.json -testing.sql \ No newline at end of file +testing.sql +edmond.sql \ No newline at end of file diff --git a/config/EXCHANGE_EXTRACT_SQL.json b/config/EXCHANGE_EXTRACT_SQL.json index acaccd1..0c85df9 100644 --- a/config/EXCHANGE_EXTRACT_SQL.json +++ b/config/EXCHANGE_EXTRACT_SQL.json @@ -34,7 +34,7 @@ "service_types" : "SELECT *, now() AS '_source_loaded_at' FROM service_types;", "states" : "SELECT *, now() AS '_source_loaded_at' FROM states;", "transactions" : "SELECT *, now() AS '_source_loaded_at' FROM transactions;", - "transaction_detail" : "SELECT *, now() AS '_source_loaded_at' FROM transaction_detail;", + "transaction_detail" : "SELECT id, transaction_id, REPLACE(REPLACE(REPLACE(product_code, '\n', ''),'|', ''),'\\\\','') AS product_code, REPLACE(REPLACE(REPLACE(product_name, '\n', ''),'|', ''),'\\\\','') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM `transaction_detail`;", "users" : "SELECT *, now() AS '_source_loaded_at' FROM users;", "user_email_verifications" : "SELECT *, now() AS '_source_loaded_at' FROM user_email_verifications;", "wallets" : "SELECT *, now() AS '_source_loaded_at' FROM wallets;" diff --git a/edmond.sql b/edmond.sql index d3be7a1..6f6f560 100644 --- a/edmond.sql +++ b/edmond.sql @@ -1,36 +1,144 @@ -use database RAW; -use schema EDMOND_DEV; -use warehouse CIEF_DW_DEV; +USE WAREHOUSE DEV_CIEF_SNOWFLAKE_WH_XS; +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE role accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."BOOKINGS"; +copy into "RAW"."EXCHANGE_MYSQL"."BOOKINGS" +from @exchange_gcs_stage +pattern=".*exchange_production_bookings.csv"; -SELECT - bookings.id AS booking_id, - bookings.status AS booking_status, - bookings.company_id AS booking_company_id, - bookings.service_id AS booking_service_id, - bookings.fix_currency_id AS booking_fix_currency_id, - bookings.fix_amount AS booking_fix_booking_amount, - dim_currency_rate_logs.CURRENCY_RATE_LOGS_SELLING_RATE AS booking_base_to_quote_convertion_rate, - bookings.conversion_currency_id AS booking_base_currency_id, - IF (bookings.fix-currency-id == conversion(base)-curr-id THEN fix-amount ELSE fix-amount/CURRENCY_RATE_LOGS_SELLING_RATE) - -- bookings.idk AS booking_base_booking_amount - bookings.convertible_currency_id AS booking_quote_currency_id, - IF (fix-currency-id == convertible(quote)-curr-id THEN fix-amount ELSE fix-amount*CURRENCY_RATE_LOGS_SELLING_RATE) AS FSFSDFDSFDSF, - -- bookings.idk AS booking_quote_booking_amount - ++ IF(base-currency = MYR THEN base -- bookings_amount_MYR - bookings.created_at, AS booking_created_at, - -- bookings.idk AS booking_expired_at - -- bookings.idk AS booking_is_expired - -- bookings.idk AS booking_approved_at - -- bookings.idk AS booking_is_approved - -- bookings.idk AS booking_completed_at +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."COMPANIES"; +copy into "RAW"."EXCHANGE_MYSQL"."COMPANIES" +from @exchange_gcs_stage +pattern=".*exchange_production_companies.csv"; -FROM RAW.EXCHANGE_MYSQL.BOOKINGS as bookings +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."CONTACTS"; +copy into "RAW"."EXCHANGE_MYSQL"."CONTACTS" +from @exchange_gcs_stage +pattern=".*exchange_production_contacts.csv"; -LEFT JOIN RAW.DBT_ZYAM_EXCHANGE_WAREHOUSE.DIM_CURRENCY_RATE_LOGS__EXCHANGE as dim_currency_rate_logs -ON (bookings.created_at > dim_currency_rate_logs .CURRENCY_RATE_LOGS_CREATED_DATE_FROM) -AND (bookings.created_at <= dim_currency_rate_logs .CURRENCY_RATE_LOGS_CREATED_DATE_TO) -AND (bookings.service_id = dim_currency_rate_logs .CURRENCY_RATE_LOGS_SERVICE_ID) -limit 1 +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."COUNTRIES"; +copy into "RAW"."EXCHANGE_MYSQL"."COUNTRIES" +from @exchange_gcs_stage +pattern=".*exchange_production_countries.csv"; +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."CURRENCIES"; +copy into "RAW"."EXCHANGE_MYSQL"."CURRENCIES" +from @exchange_gcs_stage +pattern=".*exchange_production_currencies.csv"; -IFF(bookings.fix_currency_id = bookings.base_currency_id, bookings.fix_amount, bookings.fix_amount * dim_currency_rate_logs.CURRENCY_RATE_LOGS_SELLING_RATE) \ No newline at end of file +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."CURRENCY_RATES"; +copy into "RAW"."EXCHANGE_MYSQL"."CURRENCY_RATES" +from @exchange_gcs_stage +pattern=".*exchange_production_currency_rates.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."CURRENCY_RATE_LOGS"; +copy into "RAW"."EXCHANGE_MYSQL"."CURRENCY_RATE_LOGS" +from @exchange_gcs_stage +pattern=".*exchange_production_currency_rate_logs.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."EMPLOYEES"; +copy into "RAW"."EXCHANGE_MYSQL"."EMPLOYEES" +from @exchange_gcs_stage +pattern=".*exchange_production_employees.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."GROUPS"; +copy into "RAW"."EXCHANGE_MYSQL"."GROUPS" +from @exchange_gcs_stage +pattern=".*exchange_production_groups.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."GROUP_TRANSACTIONS"; +copy into "RAW"."EXCHANGE_MYSQL"."GROUP_TRANSACTIONS" +from @exchange_gcs_stage +pattern=".*exchange_production_group_transactions.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."SEGMENTS"; +copy into "RAW"."EXCHANGE_MYSQL"."SEGMENTS" +from @exchange_gcs_stage +pattern=".*exchange_production_segments.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."SEGMENT_COMPANIES"; +copy into "RAW"."EXCHANGE_MYSQL"."SEGMENT_COMPANIES" +from @exchange_gcs_stage +pattern=".*exchange_production_segment_companies.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."SERVICE_TYPES"; +copy into "RAW"."EXCHANGE_MYSQL"."SERVICE_TYPES" +from @exchange_gcs_stage +pattern=".*exchange_production_service_types.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."TRANSACTIONS"; +copy into "RAW"."EXCHANGE_MYSQL"."TRANSACTIONS" +from @exchange_gcs_stage +pattern=".*exchange_production_transactions.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."TRANSACTION_DETAIL"; +copy into "RAW"."EXCHANGE_MYSQL"."TRANSACTION_DETAIL" +from @exchange_gcs_stage +pattern=".*exchange_production_transaction_detail.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."USERS"; +copy into "RAW"."EXCHANGE_MYSQL"."USERS" +from @exchange_gcs_stage +pattern=".*exchange_production_users.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."USER_EMAIL_VERIFICATIONS"; +copy into "RAW"."EXCHANGE_MYSQL"."USER_EMAIL_VERIFICATIONS" +from @exchange_gcs_stage +pattern=".*exchange_production_user_email_verifications.csv"; + +USE DATABASE DEV_CIEF_RAW_DB; +USE SCHEMA EXCHANGE_MYSQL; +USE ROLE accountadmin; +delete from "RAW"."EXCHANGE_MYSQL"."WALLETS"; +copy into "RAW"."EXCHANGE_MYSQL"."WALLETS" +from @exchange_gcs_stage +pattern=".*exchange_production_wallets.csv"; \ No newline at end of file