mirror of
https://gitlab.com/cief-data/dbt_cloud.git
synced 2026-08-19 04:14:00 +00:00
build exchange rate analysis
This commit is contained in:
Yam ZhengLim
@@ -0,0 +1,311 @@
|
||||
-- IMPORT
|
||||
WITH currency_exchange_rate_compilation AS (
|
||||
SELECT * FROM {{ ref('fct_exchange__currency_exchange_rate_compilation') }}
|
||||
),
|
||||
|
||||
orders AS (
|
||||
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
|
||||
WHERE service_type NOT IN ('BA')
|
||||
),
|
||||
|
||||
|
||||
-- LOGIC
|
||||
get_base_myr_value_for_comparison AS (
|
||||
-- For competitors where rate is equal for all myr range. Get C2M range and use for comparison
|
||||
SELECT DISTINCT
|
||||
end_range_myr AS comparison_base_value_myr
|
||||
|
||||
FROM currency_exchange_rate_compilation
|
||||
|
||||
WHERE
|
||||
provider = 'C2M'
|
||||
AND
|
||||
service_type = 'ALTR'
|
||||
|
||||
),
|
||||
|
||||
restructure_and_expand_table_cief_vs_world_rate AS (
|
||||
|
||||
SELECT
|
||||
t1.provider,
|
||||
t1.payment_method,
|
||||
t1.service_type,
|
||||
t1.myr_to_cny_currency_exchange_rate,
|
||||
t1.exchange_rate_date,
|
||||
|
||||
t2.provider AS competitor_name,
|
||||
t2.start_range_myr,
|
||||
t2.end_range_myr,
|
||||
t2.payment_method AS competitor_payment_method,
|
||||
t2.service_type AS competitor_service_type,
|
||||
t2.handling_fee_myr,
|
||||
t2.handling_fee_cny,
|
||||
ROUND(t2.myr_to_cny_currency_exchange_rate, 5) AS competitor_myr_to_cny_currency_exchange_rate,
|
||||
|
||||
get_base_myr_value_for_comparison.comparison_base_value_myr
|
||||
|
||||
FROM currency_exchange_rate_compilation t1
|
||||
|
||||
LEFT JOIN currency_exchange_rate_compilation t2
|
||||
ON ( t1.exchange_rate_date = t2.exchange_rate_date )
|
||||
AND t2.provider = 'WORLD RATE'
|
||||
|
||||
CROSS JOIN get_base_myr_value_for_comparison
|
||||
|
||||
WHERE
|
||||
t1.provider = 'CIEF'
|
||||
AND
|
||||
competitor_name IN ('WORLD RATE')
|
||||
|
||||
),
|
||||
|
||||
restructure_and_expand_table_cief_1688_vs_c2m_altr AS (
|
||||
|
||||
SELECT
|
||||
t1.provider,
|
||||
t1.payment_method,
|
||||
t1.service_type,
|
||||
t1.myr_to_cny_currency_exchange_rate,
|
||||
t1.exchange_rate_date,
|
||||
|
||||
t2.provider AS competitor_name,
|
||||
t2.start_range_myr,
|
||||
t2.end_range_myr,
|
||||
t2.payment_method AS competitor_payment_method,
|
||||
t2.service_type AS competitor_service_type,
|
||||
t2.handling_fee_myr,
|
||||
t2.handling_fee_cny,
|
||||
ROUND(t2.myr_to_cny_currency_exchange_rate, 5) AS competitor_myr_to_cny_currency_exchange_rate,
|
||||
t2.end_range_myr AS comparison_base_value_myr
|
||||
|
||||
FROM currency_exchange_rate_compilation t1
|
||||
|
||||
LEFT JOIN currency_exchange_rate_compilation t2
|
||||
ON ( t1.exchange_rate_date = t2.exchange_rate_date )
|
||||
AND t2.provider = 'C2M'
|
||||
AND t2.service_type = 'ALTR'
|
||||
|
||||
WHERE
|
||||
t1.provider = 'CIEF'
|
||||
AND
|
||||
competitor_name IN ('C2M')
|
||||
AND
|
||||
t1.service_type IN ('1688 PAYMENT')
|
||||
|
||||
),
|
||||
|
||||
restructure_and_expand_table_cief_1_day_transfer_vs_c2m_tbdf AS (
|
||||
|
||||
SELECT
|
||||
t1.provider,
|
||||
t1.payment_method,
|
||||
t1.service_type,
|
||||
t1.myr_to_cny_currency_exchange_rate,
|
||||
t1.exchange_rate_date,
|
||||
|
||||
t2.provider AS competitor_name,
|
||||
t2.start_range_myr,
|
||||
t2.end_range_myr,
|
||||
t2.payment_method AS competitor_payment_method,
|
||||
t2.service_type AS competitor_service_type,
|
||||
t2.handling_fee_myr,
|
||||
t2.handling_fee_cny,
|
||||
ROUND(t2.myr_to_cny_currency_exchange_rate, 5) AS competitor_myr_to_cny_currency_exchange_rate,
|
||||
get_base_myr_value_for_comparison.comparison_base_value_myr
|
||||
|
||||
FROM currency_exchange_rate_compilation t1
|
||||
|
||||
LEFT JOIN currency_exchange_rate_compilation t2
|
||||
ON ( t1.exchange_rate_date = t2.exchange_rate_date )
|
||||
AND t2.provider = 'C2M'
|
||||
AND t2.service_type = 'TBDF'
|
||||
|
||||
CROSS JOIN get_base_myr_value_for_comparison
|
||||
|
||||
WHERE
|
||||
t1.provider = 'CIEF'
|
||||
AND
|
||||
competitor_name IN ('C2M')
|
||||
AND
|
||||
t1.service_type IN ('1 DAY TRANSFER')
|
||||
|
||||
),
|
||||
|
||||
restructure_and_expand_table_cief_3_days_vs_sd AS (
|
||||
|
||||
SELECT
|
||||
t1.provider,
|
||||
t1.payment_method,
|
||||
t1.service_type,
|
||||
t1.myr_to_cny_currency_exchange_rate,
|
||||
t1.exchange_rate_date,
|
||||
|
||||
t2.provider AS competitor_name,
|
||||
t2.start_range_myr,
|
||||
t2.end_range_myr,
|
||||
t2.payment_method AS competitor_payment_method,
|
||||
t2.service_type AS competitor_service_type,
|
||||
t2.handling_fee_myr,
|
||||
t2.handling_fee_cny,
|
||||
ROUND(t2.myr_to_cny_currency_exchange_rate, 5) AS competitor_myr_to_cny_currency_exchange_rate,
|
||||
get_base_myr_value_for_comparison.comparison_base_value_myr
|
||||
|
||||
FROM currency_exchange_rate_compilation t1
|
||||
|
||||
LEFT JOIN currency_exchange_rate_compilation t2
|
||||
ON ( t1.exchange_rate_date = t2.exchange_rate_date )
|
||||
AND t2.provider = 'SD'
|
||||
|
||||
CROSS JOIN get_base_myr_value_for_comparison
|
||||
|
||||
WHERE
|
||||
t1.provider = 'CIEF'
|
||||
AND
|
||||
competitor_name IN ('SD')
|
||||
AND
|
||||
t1.service_type IN ('3 DAYS TRANSFER')
|
||||
|
||||
),
|
||||
|
||||
union_expanded_tables AS (
|
||||
|
||||
SELECT * FROM restructure_and_expand_table_cief_vs_world_rate
|
||||
UNION ALL
|
||||
SELECT * FROM restructure_and_expand_table_cief_1688_vs_c2m_altr
|
||||
UNION ALL
|
||||
SELECT * FROM restructure_and_expand_table_cief_1_day_transfer_vs_c2m_tbdf
|
||||
UNION ALL
|
||||
SELECT * FROM restructure_and_expand_table_cief_3_days_vs_sd
|
||||
|
||||
),
|
||||
|
||||
rate_comparison AS (
|
||||
|
||||
SELECT
|
||||
*,
|
||||
|
||||
( (comparison_base_value_myr / 1.02) * myr_to_cny_currency_exchange_rate ) AS cief_amount_cny,
|
||||
|
||||
CASE
|
||||
WHEN competitor_name = 'WORLD RATE'
|
||||
THEN comparison_base_value_myr * competitor_myr_to_cny_currency_exchange_rate
|
||||
WHEN competitor_name = 'C2M' AND competitor_service_type = 'ALTR'
|
||||
THEN ( (comparison_base_value_myr - handling_fee_myr) * competitor_myr_to_cny_currency_exchange_rate )
|
||||
WHEN competitor_name = 'C2M' AND competitor_service_type = 'TBDF'
|
||||
THEN comparison_base_value_myr * competitor_myr_to_cny_currency_exchange_rate
|
||||
WHEN competitor_name = 'SD'
|
||||
THEN ( comparison_base_value_myr * competitor_myr_to_cny_currency_exchange_rate ) - handling_fee_cny
|
||||
END AS competitor_amount_cny,
|
||||
|
||||
MAX(exchange_rate_date) OVER ( PARTITION BY competitor_name, payment_method, competitor_payment_method) AS latest_exchange_rate_date
|
||||
|
||||
FROM union_expanded_tables
|
||||
|
||||
),
|
||||
|
||||
order_metrics AS (
|
||||
|
||||
SELECT DISTINCT
|
||||
service_type,
|
||||
order_payment_method,
|
||||
DATE(order_created_datetime) AS order_date,
|
||||
SUM(order_quote_value) OVER (PARTITION BY DATE(order_created_datetime), service_type, order_payment_method) AS daily_sales_cny,
|
||||
SUM(order_quote_value) OVER (PARTITION BY DATE(order_created_datetime)) AS total_daily_sales_cny,
|
||||
|
||||
SUM(total_order_value_rm) OVER (PARTITION BY DATE(order_created_datetime), service_type, order_payment_method) AS daily_sales_myr,
|
||||
SUM(total_order_value_rm) OVER (PARTITION BY DATE(order_created_datetime)) AS total_daily_sales_myr
|
||||
|
||||
FROM orders
|
||||
|
||||
WHERE
|
||||
quote_currency_id = 2
|
||||
AND
|
||||
transaction_status = 'UPLOADED_BANK_SLIP'
|
||||
|
||||
),
|
||||
|
||||
exchange_rate_join_order_metrics AS (
|
||||
|
||||
SELECT
|
||||
rate_comparison.provider,
|
||||
rate_comparison.payment_method,
|
||||
rate_comparison.service_type,
|
||||
rate_comparison.myr_to_cny_currency_exchange_rate,
|
||||
rate_comparison.exchange_rate_date,
|
||||
rate_comparison.competitor_name,
|
||||
rate_comparison.start_range_myr,
|
||||
rate_comparison.end_range_myr,
|
||||
rate_comparison.competitor_payment_method,
|
||||
rate_comparison.competitor_service_type,
|
||||
rate_comparison.handling_fee_myr,
|
||||
rate_comparison.handling_fee_cny,
|
||||
rate_comparison.competitor_myr_to_cny_currency_exchange_rate,
|
||||
rate_comparison.comparison_base_value_myr,
|
||||
rate_comparison.cief_amount_cny,
|
||||
rate_comparison.competitor_amount_cny,
|
||||
rate_comparison.latest_exchange_rate_date,
|
||||
order_metrics.daily_sales_cny,
|
||||
order_metrics.total_daily_sales_cny,
|
||||
order_metrics.daily_sales_myr,
|
||||
order_metrics.total_daily_sales_myr,
|
||||
|
||||
ROW_NUMBER() OVER (PARTITION BY exchange_rate_date ORDER BY COALESCE(total_daily_sales_cny,0) DESC) AS partition_sort_idx,
|
||||
|
||||
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
|
||||
|
||||
FROM rate_comparison
|
||||
|
||||
LEFT JOIN order_metrics
|
||||
ON
|
||||
(
|
||||
rate_comparison.exchange_rate_date = order_metrics.order_date
|
||||
AND rate_comparison.service_type = order_metrics.service_type
|
||||
AND rate_comparison.payment_method = order_metrics.order_payment_method
|
||||
)
|
||||
|
||||
),
|
||||
|
||||
|
||||
-- FINAL
|
||||
final__rep_exchange__exchange_rate_analysis AS (
|
||||
|
||||
SELECT
|
||||
|
||||
-- ids
|
||||
partition_sort_idx,
|
||||
|
||||
-- dimensions
|
||||
provider,
|
||||
service_type,
|
||||
payment_method,
|
||||
competitor_name,
|
||||
competitor_service_type,
|
||||
competitor_payment_method,
|
||||
|
||||
-- measures
|
||||
start_range_myr,
|
||||
end_range_myr,
|
||||
comparison_base_value_myr,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
competitor_myr_to_cny_currency_exchange_rate,
|
||||
handling_fee_myr,
|
||||
handling_fee_cny,
|
||||
cief_amount_cny,
|
||||
competitor_amount_cny,
|
||||
daily_sales_cny,
|
||||
total_daily_sales_cny,
|
||||
daily_sales_myr,
|
||||
total_daily_sales_myr,
|
||||
|
||||
-- date/times
|
||||
exchange_rate_date,
|
||||
latest_exchange_rate_date,
|
||||
|
||||
-- metadata
|
||||
_dbt_ran_datetime
|
||||
|
||||
FROM exchange_rate_join_order_metrics
|
||||
|
||||
)
|
||||
|
||||
SELECT * FROM final__rep_exchange__exchange_rate_analysis
|
||||
@@ -0,0 +1,238 @@
|
||||
-- IMPORT
|
||||
WITH open_api_exchange_rate_logs AS (
|
||||
SELECT * FROM {{ ref('stg_exchange__open_api_currency_rate_logs') }}
|
||||
),
|
||||
|
||||
cief_exchange_rate_logs AS (
|
||||
SELECT * FROM {{ ref('stg_exchange__currency_rate_logs') }}
|
||||
),
|
||||
|
||||
c2m_exchange_rate_logs AS (
|
||||
SELECT * FROM {{ ref('stg_exchange__c2m_currency_rate_logs') }}
|
||||
),
|
||||
|
||||
sd_exchange_rate_logs AS (
|
||||
SELECT * FROM {{ ref('stg_googlesheet__sd_currency_exchange_rates') }}
|
||||
),
|
||||
|
||||
|
||||
-- LOGIC
|
||||
-- CIEF Rates
|
||||
cief_exchange_rate_log_filtering AS (
|
||||
|
||||
SELECT
|
||||
*
|
||||
|
||||
FROM cief_exchange_rate_logs
|
||||
|
||||
WHERE
|
||||
payment_method NOT IN ('BA')
|
||||
AND
|
||||
payment_method IS NOT NULL
|
||||
AND
|
||||
service_type IN ('1 DAY TRANSFER', '3 DAYS TRANSFER', 'ENTERPRISE TO ENTERPRISE 公打公', '1688 PAYMENT')
|
||||
AND
|
||||
quote_currency_id = 2
|
||||
|
||||
),
|
||||
|
||||
cief_exchange_rates AS (
|
||||
|
||||
SELECT
|
||||
'CIEF' AS provider,
|
||||
1 AS start_range_myr,
|
||||
200000 AS end_range_myr,
|
||||
payment_method,
|
||||
service_type,
|
||||
NULL AS handling_fee_myr,
|
||||
NULL AS handling_fee_cny,
|
||||
base_to_quote_currency_exchange_rate AS myr_to_cny_currency_exchange_rate,
|
||||
|
||||
open_api_exchange_rate_logs.exchange_rate_date
|
||||
|
||||
FROM cief_exchange_rate_log_filtering
|
||||
|
||||
INNER JOIN open_api_exchange_rate_logs
|
||||
ON ( open_api_exchange_rate_logs.exchange_rate_date >= cief_exchange_rate_log_filtering.from_datetime )
|
||||
AND (open_api_exchange_rate_logs.exchange_rate_date < cief_exchange_rate_log_filtering.to_datetime)
|
||||
|
||||
),
|
||||
|
||||
-- World Rates
|
||||
world_rate_exchange_rates AS (
|
||||
|
||||
SELECT
|
||||
'WORLD RATE' AS provider,
|
||||
1 AS start_range_myr,
|
||||
200000 AS end_range_myr,
|
||||
NULL AS payment_method,
|
||||
NULL AS service_type,
|
||||
NULL AS handling_fee_myr,
|
||||
NULL AS handling_fee_cny,
|
||||
myr_to_cny_rate AS myr_to_cny_currency_exchange_rate,
|
||||
exchange_rate_date
|
||||
|
||||
FROM open_api_exchange_rate_logs
|
||||
|
||||
),
|
||||
|
||||
-- C2M Rates
|
||||
c2m_ranked_values AS (
|
||||
|
||||
SELECT
|
||||
base_value_myr,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
handling_fee_myr,
|
||||
created_datetime,
|
||||
service_type,
|
||||
LAG(base_value_myr, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr ) AS prev_base_value_myr,
|
||||
LEAD(base_value_myr, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr ) AS next_base_value_myr,
|
||||
|
||||
LAG(myr_to_cny_currency_exchange_rate, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr) AS prev_rate,
|
||||
LEAD(myr_to_cny_currency_exchange_rate, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr) AS next_rate,
|
||||
|
||||
LAG(handling_fee_myr, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr) AS prev_fee,
|
||||
LEAD(handling_fee_myr, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr) AS next_fee
|
||||
|
||||
FROM c2m_exchange_rate_logs
|
||||
|
||||
),
|
||||
|
||||
c2m_rate_changed_rows AS (
|
||||
|
||||
SELECT
|
||||
base_value_myr,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
handling_fee_myr,
|
||||
created_datetime,
|
||||
service_type,
|
||||
|
||||
CASE
|
||||
WHEN LAG(myr_to_cny_currency_exchange_rate) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr) != myr_to_cny_currency_exchange_rate
|
||||
OR LAG(handling_fee_myr) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr) != handling_fee_myr
|
||||
THEN 1
|
||||
ELSE 0
|
||||
END AS is_rate_change_row
|
||||
|
||||
FROM c2m_exchange_rate_logs
|
||||
|
||||
),
|
||||
|
||||
c2m_group_changed_rows AS (
|
||||
|
||||
SELECT
|
||||
base_value_myr,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
handling_fee_myr,
|
||||
created_datetime,
|
||||
service_type,
|
||||
SUM(is_rate_change_row) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr) AS group_id,
|
||||
LEAD(base_value_myr, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr ) - 1 AS next_base_value_myr
|
||||
|
||||
FROM c2m_rate_changed_rows
|
||||
|
||||
),
|
||||
|
||||
c2m_base_value_ranges AS (
|
||||
|
||||
SELECT
|
||||
'C2M' AS provider,
|
||||
MIN(base_value_myr) AS start_range_myr,
|
||||
COALESCE( MAX(next_base_value_myr), 200000 ) AS end_range_myr,
|
||||
NULL AS payment_method,
|
||||
service_type,
|
||||
handling_fee_myr,
|
||||
NULL AS handling_fee_cny,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
DATE(created_datetime) AS exchange_rate_date
|
||||
|
||||
FROM c2m_group_changed_rows
|
||||
|
||||
GROUP BY
|
||||
service_type,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
handling_fee_myr,
|
||||
DATE(created_datetime),
|
||||
group_id
|
||||
|
||||
),
|
||||
|
||||
-- SD Group Rates
|
||||
sd_exchange_rates AS (
|
||||
|
||||
SELECT
|
||||
'SD' AS provider,
|
||||
1 AS start_range_myr,
|
||||
200000 AS end_range_myr,
|
||||
payment_method,
|
||||
NULL AS service_type,
|
||||
NULL AS handling_fee_myr,
|
||||
handling_fee_cny,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
exchange_rate_date
|
||||
|
||||
FROM sd_exchange_rate_logs
|
||||
|
||||
),
|
||||
|
||||
union_competitor_rates AS (
|
||||
|
||||
SELECT * FROM world_rate_exchange_rates
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT * FROM cief_exchange_rates
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT * FROM c2m_base_value_ranges
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT * FROM sd_exchange_rates
|
||||
|
||||
),
|
||||
|
||||
add_metadata AS (
|
||||
|
||||
SELECT
|
||||
*,
|
||||
|
||||
MD5_NUMBER_LOWER64( CONCAT(provider, myr_to_cny_currency_exchange_rate, exchange_rate_date) ) AS exchange_rate_log_id,
|
||||
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
|
||||
|
||||
FROM union_competitor_rates
|
||||
|
||||
),
|
||||
|
||||
|
||||
-- FINAL
|
||||
final__fct_exchange__currency_exchange_rate_compilation AS (
|
||||
|
||||
SELECT
|
||||
--id
|
||||
exchange_rate_log_id,
|
||||
|
||||
--dimension
|
||||
provider,
|
||||
payment_method,
|
||||
service_type,
|
||||
|
||||
--measures
|
||||
start_range_myr,
|
||||
end_range_myr,
|
||||
handling_fee_myr,
|
||||
handling_fee_cny,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
|
||||
--date/time
|
||||
exchange_rate_date,
|
||||
|
||||
--metadata
|
||||
_dbt_ran_datetime
|
||||
|
||||
FROM add_metadata
|
||||
|
||||
)
|
||||
|
||||
SELECT * FROM final__fct_exchange__currency_exchange_rate_compilation
|
||||
@@ -0,0 +1,29 @@
|
||||
version: 2
|
||||
|
||||
sources:
|
||||
- name: src_c2m_exchange_rates
|
||||
description: Currency exchange rate data of competitor C2M, obtained from web scraping. Link=https://www.c2m.my/
|
||||
database: DEV_CIEF_RAW_DB
|
||||
schema: C2M_WEBSCRAPE
|
||||
loader: custom_python_code
|
||||
loaded_at_field: SCRAPE_DATE
|
||||
freshness:
|
||||
warn_after: {count: 26, period: hour}
|
||||
error_after: {count: 48, period: hour}
|
||||
meta:
|
||||
owner: "@yam"
|
||||
model_maturity: prod
|
||||
tags:
|
||||
- exchange_rate_log
|
||||
- daily
|
||||
|
||||
|
||||
tables:
|
||||
- name: currency_rate_logs
|
||||
description: Web scraped daily currency exchange rates of different products offered by competitor C2M.
|
||||
columns:
|
||||
- name: service_type
|
||||
description: (
|
||||
ALTR:ALIPAY:Transfer/Scan-To-Pay,
|
||||
TBDF:TAOBAO/1688:Payment-On-Behalf
|
||||
)
|
||||
@@ -0,0 +1,114 @@
|
||||
-- IMPORT
|
||||
WITH c2m_exchange_rate_log AS (
|
||||
SELECT * FROM {{ source('src_c2m_exchange_rates', 'currency_rate_logs') }}
|
||||
),
|
||||
|
||||
|
||||
-- LOGIC
|
||||
data_cleaning AS (
|
||||
SELECT
|
||||
*,
|
||||
|
||||
-- remove handling fee for TBDF
|
||||
CASE
|
||||
WHEN service_type = 'TBDF'
|
||||
THEN 0
|
||||
ELSE handeling_fee
|
||||
END AS cleaned_handling_fee
|
||||
|
||||
FROM c2m_exchange_rate_log
|
||||
),
|
||||
|
||||
rename_and_casting AS (
|
||||
|
||||
SELECT
|
||||
product_name,
|
||||
site_name AS competitor_name,
|
||||
service_type,
|
||||
pending_order AS order_queue_number,
|
||||
myr::FLOAT AS base_value_myr,
|
||||
cny::FLOAT AS quote_value_cny,
|
||||
cleaned_handling_fee::FLOAT AS handling_fee_myr,
|
||||
REPLACE(payable, ',','')::FLOAT AS payable_myr,
|
||||
current_rate::FLOAT AS myr_to_cny_currency_exchange_rate,
|
||||
staff_status,
|
||||
DATEADD(DAY, -1, scrape_date) AS created_datetime,
|
||||
scrape_hour,
|
||||
|
||||
TO_TIMESTAMP(
|
||||
REGEXP_SUBSTR(
|
||||
last_rate_update,
|
||||
'[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{1,2}:[0-9]{2} [APap][Mm]'
|
||||
),
|
||||
'YYYY-MM-DD HH12:MI AM'
|
||||
) AS rate_updated_datetime
|
||||
|
||||
FROM data_cleaning
|
||||
|
||||
),
|
||||
|
||||
recalculate_payable_column AS (
|
||||
|
||||
SELECT
|
||||
MD5_NUMBER_LOWER64(CONCAT(service_type, base_value_myr, created_datetime )) AS currency_rate_log_id,
|
||||
|
||||
product_name,
|
||||
competitor_name,
|
||||
service_type,
|
||||
order_queue_number,
|
||||
base_value_myr,
|
||||
quote_value_cny,
|
||||
handling_fee_myr,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
staff_status,
|
||||
created_datetime,
|
||||
scrape_hour,
|
||||
rate_updated_datetime,
|
||||
|
||||
-- Recalculate payable myr after removing handling fee
|
||||
CASE
|
||||
WHEN service_type = 'TBDF'
|
||||
THEN base_value_myr
|
||||
ELSE payable_myr
|
||||
END AS payable_myr,
|
||||
|
||||
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
|
||||
|
||||
FROM rename_and_casting
|
||||
|
||||
),
|
||||
|
||||
|
||||
-- FINAL
|
||||
final__stg_exchange__c2m_exchange_rates AS (
|
||||
|
||||
SELECT
|
||||
-- ids
|
||||
currency_rate_log_id,
|
||||
|
||||
-- dimensions
|
||||
competitor_name,
|
||||
product_name,
|
||||
service_type,
|
||||
staff_status,
|
||||
|
||||
-- measures
|
||||
order_queue_number,
|
||||
base_value_myr,
|
||||
quote_value_cny,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
handling_fee_myr,
|
||||
payable_myr,
|
||||
|
||||
-- date/time
|
||||
rate_updated_datetime,
|
||||
created_datetime,
|
||||
|
||||
-- metadata
|
||||
_dbt_ran_datetime
|
||||
|
||||
FROM recalculate_payable_column
|
||||
|
||||
)
|
||||
|
||||
SELECT * FROM final__stg_exchange__c2m_exchange_rates
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
version: 2
|
||||
|
||||
sources:
|
||||
- name: src_open_api_exchange_rates
|
||||
description: Latest currency exchange rates of multiple currencies obtained from `Open Exchange Rates` API call.
|
||||
database: DEV_CIEF_RAW_DB
|
||||
schema: OPENEXCHANGERATES_AIRBYTE
|
||||
loader: custom_python_code
|
||||
loaded_at_field: _airbyte_extracted_at
|
||||
freshness:
|
||||
warn_after: {count: 26, period: hour}
|
||||
error_after: {count: 48, period: hour}
|
||||
meta:
|
||||
owner: "@yam"
|
||||
model_maturity: prod
|
||||
tags:
|
||||
- exchange_rate_log
|
||||
- daily
|
||||
|
||||
|
||||
tables:
|
||||
- name: current_rate
|
||||
description: Hourly currency exchange rates obtained from the API call.
|
||||
columns:
|
||||
- name: _airbyte_raw_id
|
||||
description: Primary key for 'current_rate'
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
-- IMPORT
|
||||
WITH api_call_exchange_rates AS (
|
||||
SELECT * FROM {{ source('src_open_api_exchange_rates', 'current_rate') }}
|
||||
),
|
||||
|
||||
seed_historical_exchange_rates AS (
|
||||
SELECT * FROM {{ ref('seed_exchange__historical_currency_exchange_rates') }}
|
||||
),
|
||||
|
||||
|
||||
-- LOGIC
|
||||
api_rename_and_casting AS (
|
||||
|
||||
SELECT
|
||||
_airbyte_raw_id AS _airbyte_id,
|
||||
base AS base_currency,
|
||||
PARSE_JSON(rates::STRING) AS parsed_rates,
|
||||
_airbyte_extracted_at AS _airbyte_extracted_datetime,
|
||||
TO_TIMESTAMP_NTZ(timestamp::INT) AS exchange_rate_datetime
|
||||
|
||||
FROM api_call_exchange_rates
|
||||
|
||||
),
|
||||
|
||||
api_currency_conversion AS (
|
||||
|
||||
SELECT
|
||||
DATE(exchange_rate_datetime) AS exchange_rate_date,
|
||||
AVG(1 / parsed_rates['MYR']) AS myr_to_usd_rate,
|
||||
AVG(parsed_rates['CNY'] / parsed_rates['MYR']) AS myr_to_cny_rate
|
||||
|
||||
FROM api_rename_and_casting
|
||||
|
||||
GROUP BY
|
||||
DATE(exchange_rate_datetime)
|
||||
|
||||
),
|
||||
|
||||
seed_currency_conversion AS (
|
||||
|
||||
SELECT
|
||||
TO_CHAR(TO_DATE(date,'DD-MM-YYYY'), 'YYYY-MM-DD') AS exchange_rate_date,
|
||||
cny / myr AS myr_to_cny_rate,
|
||||
1 / myr AS myr_to_usd_rate
|
||||
|
||||
FROM seed_historical_exchange_rates
|
||||
|
||||
),
|
||||
|
||||
union_exchange_rates AS (
|
||||
|
||||
SELECT
|
||||
exchange_rate_date,
|
||||
myr_to_cny_rate,
|
||||
myr_to_usd_rate
|
||||
|
||||
FROM seed_currency_conversion
|
||||
|
||||
WHERE
|
||||
exchange_rate_date < '2024-01-04'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
exchange_rate_date,
|
||||
myr_to_cny_rate,
|
||||
myr_to_usd_rate
|
||||
|
||||
FROM api_currency_conversion
|
||||
|
||||
),
|
||||
|
||||
add_primary_key_column AS (
|
||||
|
||||
SELECT
|
||||
*,
|
||||
MD5_NUMBER_LOWER64(CONCAT(myr_to_cny_rate, myr_to_usd_rate, exchange_rate_date)) AS currency_rate_log_id,
|
||||
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
|
||||
|
||||
FROM union_exchange_rates
|
||||
|
||||
),
|
||||
|
||||
|
||||
-- FINAL
|
||||
final__stg_exchange__api_call_exchange_rates AS (
|
||||
|
||||
SELECT
|
||||
-- id
|
||||
currency_rate_log_id,
|
||||
|
||||
-- dimension
|
||||
|
||||
-- measure
|
||||
myr_to_usd_rate,
|
||||
myr_to_cny_rate,
|
||||
|
||||
-- date/time
|
||||
exchange_rate_date,
|
||||
|
||||
-- metadata
|
||||
_dbt_ran_datetime
|
||||
|
||||
FROM add_primary_key_column
|
||||
|
||||
)
|
||||
|
||||
SELECT * FROM final__stg_exchange__api_call_exchange_rates
|
||||
@@ -44,6 +44,16 @@ sources:
|
||||
columns:
|
||||
- name: _airbyte_raw_id
|
||||
description: Primary key for 'account_frozen_case'
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
|
||||
|
||||
- name: sd_currency_exchange_rates
|
||||
description: Data retrieved from gsheet, comprises the currency exchange rates from our competitor SD.
|
||||
columns:
|
||||
- name: _airbyte_raw_id
|
||||
description: Primary key for 'sd_currency_exchange_rates'
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
@@ -0,0 +1,71 @@
|
||||
-- IMPORT
|
||||
WITH sd_exchange_rate_log AS (
|
||||
SELECT * FROM {{ source('src_googlesheet_airbyte', 'sd_currency_exchange_rates') }}
|
||||
),
|
||||
|
||||
seed_metadata AS (
|
||||
SELECT * FROM {{ ref('seed_exchange__metadata') }}
|
||||
),
|
||||
|
||||
|
||||
-- LOGIC
|
||||
rename_and_casting AS (
|
||||
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(['payment_method', 'myr_to_cny_rate', 'exchange_rate_date']) }} AS sd_exchange_rate_id,
|
||||
|
||||
_airbyte_raw_id AS _airbyte_id,
|
||||
|
||||
sd_exchange_rate_log.provider AS competitor_name,
|
||||
sd_exchange_rate_log.payment_method,
|
||||
sd_exchange_rate_log.myr_to_cny_rate::FLOAT AS myr_to_cny_currency_exchange_rate,
|
||||
sd_exchange_rate_log.transfer_days AS transfer_duration_remark,
|
||||
sd_exchange_rate_log.remark,
|
||||
|
||||
COALESCE(
|
||||
TRY_TO_DATE(sd_exchange_rate_log.exchange_rate_date, 'DD-MM-YY'),
|
||||
TRY_TO_DATE(sd_exchange_rate_log.exchange_rate_date, 'MM-DD-YY')
|
||||
) AS exchange_rate_date,
|
||||
|
||||
seed_metadata.value AS handling_fee_cny,
|
||||
|
||||
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
|
||||
|
||||
FROM sd_exchange_rate_log
|
||||
|
||||
LEFT JOIN seed_metadata
|
||||
ON (seed_metadata.id = 1)
|
||||
-- refer to model name when referencing from seed
|
||||
-- But {{this.name}} returning 'inline_query'
|
||||
|
||||
),
|
||||
|
||||
|
||||
-- FINAL
|
||||
final__stg_exchange__sd_currency_rate_logs AS (
|
||||
|
||||
SELECT
|
||||
-- ids
|
||||
sd_exchange_rate_id,
|
||||
|
||||
-- dimensions
|
||||
competitor_name,
|
||||
payment_method,
|
||||
transfer_duration_remark,
|
||||
remark,
|
||||
|
||||
-- measures
|
||||
handling_fee_cny,
|
||||
myr_to_cny_currency_exchange_rate,
|
||||
|
||||
-- date/time
|
||||
exchange_rate_date,
|
||||
|
||||
-- metadata
|
||||
_dbt_ran_datetime
|
||||
|
||||
FROM rename_and_casting
|
||||
|
||||
)
|
||||
|
||||
SELECT * FROM final__stg_exchange__sd_currency_rate_logs
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,2 @@
|
||||
ID,TABLE_NAME,SETTING_NAME,COLUMN_NAME,LOGIC,VALUE,LAST_UPDATED_DATE
|
||||
1,stg_exchange__sd_currency_rate_logs,sd have 20CNY charge every transaction,handling_fee_cny,add column,20,2024-02-20
|
||||
|
Reference in New Issue
Block a user