diff --git a/dbt_project.yml b/dbt_project.yml index d503f08..3cda563 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -72,7 +72,17 @@ models: base: +materialized: view - +schema: shipping__base + +schema: shipping__base + + currency_exchange_rates: + c2m: + +materialized: view + +schema: currency_exchange_rate__c2m + + world_rate: + +materialized: view + +schema: currency_exchange_rate__open_exchange_api + intermediate: +materialized: ephemeral diff --git a/models/marts/warehouse/fct_exchange__currency_exchange_rate_compilation.sql b/models/marts/warehouse/fct_exchange__currency_exchange_rate_compilation.sql index 4e1e3a2..d2db6da 100644 --- a/models/marts/warehouse/fct_exchange__currency_exchange_rate_compilation.sql +++ b/models/marts/warehouse/fct_exchange__currency_exchange_rate_compilation.sql @@ -1,6 +1,6 @@ -- IMPORT WITH open_api_exchange_rate_logs AS ( - SELECT * FROM {{ ref('stg_exchange__open_api_currency_rate_logs') }} + SELECT * FROM {{ ref('stg_open_ex_api__open_api_currency_rate_logs') }} ), cief_exchange_rate_logs AS ( @@ -8,7 +8,7 @@ cief_exchange_rate_logs AS ( ), c2m_exchange_rate_logs AS ( - SELECT * FROM {{ ref('stg_exchange__c2m_currency_rate_logs') }} + SELECT * FROM {{ ref('stg_c2m_exchange__c2m_currency_rate_logs') }} ), sd_exchange_rate_logs AS ( @@ -28,10 +28,6 @@ cief_exchange_rate_log_filtering AS ( WHERE payment_method NOT IN ('BA') AND - payment_method IS NOT NULL - AND - service_type IN ('1 DAY TRANSFER', '3 DAYS TRANSFER', 'ENTERPRISE TO ENTERPRISE 公打公', '1688 PAYMENT') - AND quote_currency_id = 2 ), diff --git a/models/staging/currency_exchange_rates/c2m/stg_exchange__c2m_currency_rate_logs.sql b/models/staging/currency_exchange_rates/c2m/stg_c2m_exchange__c2m_currency_rate_logs.sql similarity index 100% rename from models/staging/currency_exchange_rates/c2m/stg_exchange__c2m_currency_rate_logs.sql rename to models/staging/currency_exchange_rates/c2m/stg_c2m_exchange__c2m_currency_rate_logs.sql diff --git a/models/staging/currency_exchange_rates/world_rate/stg_exchange__open_api_currency_rate_logs.sql b/models/staging/currency_exchange_rates/world_rate/stg_open_ex_api__open_api_currency_rate_logs.sql similarity index 100% rename from models/staging/currency_exchange_rates/world_rate/stg_exchange__open_api_currency_rate_logs.sql rename to models/staging/currency_exchange_rates/world_rate/stg_open_ex_api__open_api_currency_rate_logs.sql diff --git a/models/staging/exchange/stg_exchange__currency_rate_logs.sql b/models/staging/exchange/stg_exchange__currency_rate_logs.sql index 5dd8e8a..e2788c6 100644 --- a/models/staging/exchange/stg_exchange__currency_rate_logs.sql +++ b/models/staging/exchange/stg_exchange__currency_rate_logs.sql @@ -41,9 +41,9 @@ currency_rate_logs_join_currency_rates AS ( LEFT JOIN currency_rates ON (currency_rate_logs.currency_rate_id = currency_rates.currency_rate_id) + ), - currency_rate_logs_quote_currency_id_join_currencies AS ( SELECT @@ -58,7 +58,7 @@ currency_rate_logs_quote_currency_id_join_currencies AS ( currency_rate_logs_join_currency_rates.base_to_quote_currency_exchange_rate, COALESCE(payment_methods.name, currency_rate_logs_join_currency_rates.payment_method::string) AS payment_method, - COALESCE(service_types.name, currency_rate_logs_join_currency_rates.service_type::string) AS service_type, + COALESCE(service_types.name, currency_rate_logs_join_currency_rates.service_type::string) AS service_type, currency_rate_logs_join_currency_rates.creator_user_id, currency_rate_logs_join_currency_rates.created_datetime, @@ -75,66 +75,81 @@ currency_rate_logs_quote_currency_id_join_currencies AS ( LEFT JOIN service_types ON (currency_rate_logs_join_currency_rates.service_type = service_types.service_type_id) + +), + +filtered_currency_rate_logs AS ( + + SELECT + * + + FROM currency_rate_logs_quote_currency_id_join_currencies + + WHERE service_type NOT IN ('x1', 'Company to Company', 'RECEIVED-ON-BEHALF') + ), currency_rate_logs_add_dense_rank AS ( SELECT - currency_rate_logs_quote_currency_id_join_currencies.currency_rate_log_id, + filtered_currency_rate_logs.currency_rate_log_id, - dense_rank() OVER - (PARTITION BY currency_rate_logs_quote_currency_id_join_currencies.currency_rate_id - ORDER BY currency_rate_logs_quote_currency_id_join_currencies.created_datetime) AS dense_rank_index, + DENSE_RANK() OVER + (PARTITION BY filtered_currency_rate_logs.currency_rate_id + ORDER BY filtered_currency_rate_logs.created_datetime) AS dense_rank_index, - currency_rate_logs_quote_currency_id_join_currencies.currency_rate_id AS currency_rate_id, - currency_rate_logs_quote_currency_id_join_currencies.quote_currency_id, - currency_rate_logs_quote_currency_id_join_currencies.quote_currency_name, - currency_rate_logs_quote_currency_id_join_currencies.quote_currency_short_code, - currency_rate_logs_quote_currency_id_join_currencies.quote_currency_symbol, - currency_rate_logs_quote_currency_id_join_currencies.payment_method, - currency_rate_logs_quote_currency_id_join_currencies.service_type, - currency_rate_logs_quote_currency_id_join_currencies.base_to_quote_currency_exchange_rate, - currency_rate_logs_quote_currency_id_join_currencies.creator_user_id, - currency_rate_logs_quote_currency_id_join_currencies.created_datetime + filtered_currency_rate_logs.currency_rate_id AS currency_rate_id, + filtered_currency_rate_logs.quote_currency_id, + filtered_currency_rate_logs.quote_currency_name, + filtered_currency_rate_logs.quote_currency_short_code, + filtered_currency_rate_logs.quote_currency_symbol, + filtered_currency_rate_logs.payment_method, + filtered_currency_rate_logs.service_type, + filtered_currency_rate_logs.base_to_quote_currency_exchange_rate, + filtered_currency_rate_logs.creator_user_id, + filtered_currency_rate_logs.created_datetime - FROM currency_rate_logs_quote_currency_id_join_currencies + FROM filtered_currency_rate_logs ORDER BY currency_rate_id, created_datetime + ), +currency_rate_logs_resturucure AS ( -currency_rate_logs_resturucure AS ( - SELECT - currency_rate_logs_add_dense_rank.currency_rate_log_id, - currency_rate_logs_add_dense_rank.currency_rate_id, - currency_rate_logs_add_dense_rank.quote_currency_id, - currency_rate_logs_add_dense_rank.quote_currency_name, - currency_rate_logs_add_dense_rank.quote_currency_short_code, - currency_rate_logs_add_dense_rank.quote_currency_symbol, - currency_rate_logs_add_dense_rank.payment_method, - currency_rate_logs_add_dense_rank.service_type, - currency_rate_logs_add_dense_rank.base_to_quote_currency_exchange_rate, - currency_rate_logs_add_dense_rank.creator_user_id, + SELECT + currency_rate_logs_add_dense_rank.currency_rate_log_id, + currency_rate_logs_add_dense_rank.currency_rate_id, + currency_rate_logs_add_dense_rank.quote_currency_id, + currency_rate_logs_add_dense_rank.quote_currency_name, + currency_rate_logs_add_dense_rank.quote_currency_short_code, + currency_rate_logs_add_dense_rank.quote_currency_symbol, + currency_rate_logs_add_dense_rank.payment_method, + currency_rate_logs_add_dense_rank.service_type, + currency_rate_logs_add_dense_rank.base_to_quote_currency_exchange_rate, + currency_rate_logs_add_dense_rank.creator_user_id, - CASE - WHEN currency_rate_logs_add_dense_rank.dense_rank_index=1 THEN CAST('1990-01-01 00:00:00 +08:00' AS DATETIME) - ELSE currency_rate_logs_add_dense_rank.created_datetime - END AS from_datetime, + CASE + WHEN currency_rate_logs_add_dense_rank.dense_rank_index=1 THEN CAST('1990-01-01 00:00:00 +08:00' AS DATETIME) + ELSE currency_rate_logs_add_dense_rank.created_datetime + END AS from_datetime, - COALESCE(currency_rate_logs_add_dense_rank_clone.created_datetime, - CAST('2999-12-31 23:59:59 +08:00' AS DATETIME)) AS to_datetime, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + COALESCE(currency_rate_logs_add_dense_rank_clone.created_datetime, + CAST('2999-12-31 23:59:59 +08:00' AS DATETIME)) AS to_datetime, - FROM - currency_rate_logs_add_dense_rank + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - LEFT JOIN currency_rate_logs_add_dense_rank AS currency_rate_logs_add_dense_rank_clone - ON currency_rate_logs_add_dense_rank.dense_rank_index = currency_rate_logs_add_dense_rank_clone.dense_rank_index-1 - AND currency_rate_logs_add_dense_rank.currency_rate_id = currency_rate_logs_add_dense_rank_clone.currency_rate_id + FROM + currency_rate_logs_add_dense_rank + + LEFT JOIN currency_rate_logs_add_dense_rank AS currency_rate_logs_add_dense_rank_clone + ON currency_rate_logs_add_dense_rank.dense_rank_index = currency_rate_logs_add_dense_rank_clone.dense_rank_index-1 + AND currency_rate_logs_add_dense_rank.currency_rate_id = currency_rate_logs_add_dense_rank_clone.currency_rate_id + + ORDER BY currency_rate_logs_add_dense_rank.currency_rate_id, currency_rate_logs_add_dense_rank.created_datetime - ORDER BY currency_rate_logs_add_dense_rank.currency_rate_id, currency_rate_logs_add_dense_rank.created_datetime ),