Merge branch 'main' of gitlab.com:cief-data/dbt_cloud into lee_build_exchange_rate_fact

dbt_project.yml

@@ -72,7 +72,17 @@ models:
             
           base:
             +materialized: view
-            +schema: shipping__base              
+            +schema: shipping__base
+
+      currency_exchange_rates:
+          c2m:
+            +materialized: view
+            +schema: currency_exchange_rate__c2m
+
+          world_rate:
+            +materialized: view
+            +schema: currency_exchange_rate__open_exchange_api
+            
     
     intermediate:
       +materialized: ephemeral

models/marts/reporting/rep_exchange__exchange_rate_analysis.sql

@@ -0,0 +1,311 @@
+-- IMPORT
+WITH currency_exchange_rate_compilation AS (
+    SELECT * FROM {{ ref('fct_exchange__currency_exchange_rate_compilation') }}
+),
+
+orders AS (
+    SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
+    WHERE service_type NOT IN ('BA')
+),
+
+
+-- LOGIC
+get_base_myr_value_for_comparison AS (
+    -- For competitors where rate is equal for all myr range. Get C2M range and use for comparison
+    SELECT DISTINCT 
+        end_range_myr AS comparison_base_value_myr
+
+    FROM currency_exchange_rate_compilation
+
+    WHERE 
+        provider = 'C2M' 
+        AND 
+        service_type = 'ALTR'
+
+),
+
+restructure_and_expand_table_cief_vs_world_rate AS (
+    
+    SELECT
+        t1.provider,
+        t1.payment_method,
+        t1.service_type,
+        t1.myr_to_cny_currency_exchange_rate,
+        t1.exchange_rate_date,
+
+        t2.provider                 AS competitor_name,
+        t2.start_range_myr,
+        t2.end_range_myr,
+        t2.payment_method           AS competitor_payment_method,
+        t2.service_type             AS competitor_service_type,
+        t2.handling_fee_myr,
+        t2.handling_fee_cny,
+        ROUND(t2.myr_to_cny_currency_exchange_rate, 5)    AS competitor_myr_to_cny_currency_exchange_rate,
+
+        get_base_myr_value_for_comparison.comparison_base_value_myr
+
+    FROM currency_exchange_rate_compilation t1
+
+    LEFT JOIN currency_exchange_rate_compilation t2
+        ON ( t1.exchange_rate_date = t2.exchange_rate_date )
+        AND t2.provider = 'WORLD RATE'
+
+    CROSS JOIN get_base_myr_value_for_comparison
+
+    WHERE 
+        t1.provider = 'CIEF'
+        AND
+        competitor_name IN ('WORLD RATE')
+
+),
+
+restructure_and_expand_table_cief_1688_vs_c2m_altr AS (
+    
+    SELECT
+        t1.provider,
+        t1.payment_method,
+        t1.service_type,
+        t1.myr_to_cny_currency_exchange_rate,
+        t1.exchange_rate_date,
+
+        t2.provider                 AS competitor_name,
+        t2.start_range_myr,
+        t2.end_range_myr,
+        t2.payment_method           AS competitor_payment_method,
+        t2.service_type             AS competitor_service_type,
+        t2.handling_fee_myr,
+        t2.handling_fee_cny,
+        ROUND(t2.myr_to_cny_currency_exchange_rate, 5)      AS competitor_myr_to_cny_currency_exchange_rate,
+        t2.end_range_myr                                    AS comparison_base_value_myr
+
+    FROM currency_exchange_rate_compilation t1
+
+    LEFT JOIN currency_exchange_rate_compilation t2
+        ON ( t1.exchange_rate_date = t2.exchange_rate_date )
+        AND t2.provider = 'C2M'
+        AND t2.service_type = 'ALTR'
+
+    WHERE 
+        t1.provider = 'CIEF'
+        AND
+        competitor_name IN ('C2M')
+        AND 
+        t1.service_type IN ('1688 PAYMENT')
+
+),
+
+restructure_and_expand_table_cief_1_day_transfer_vs_c2m_tbdf AS (
+    
+    SELECT
+        t1.provider,
+        t1.payment_method,
+        t1.service_type,
+        t1.myr_to_cny_currency_exchange_rate,
+        t1.exchange_rate_date,
+
+        t2.provider                 AS competitor_name,
+        t2.start_range_myr,
+        t2.end_range_myr,
+        t2.payment_method           AS competitor_payment_method,
+        t2.service_type             AS competitor_service_type,
+        t2.handling_fee_myr,
+        t2.handling_fee_cny,
+        ROUND(t2.myr_to_cny_currency_exchange_rate, 5)      AS competitor_myr_to_cny_currency_exchange_rate,
+        get_base_myr_value_for_comparison.comparison_base_value_myr
+
+    FROM currency_exchange_rate_compilation t1
+
+    LEFT JOIN currency_exchange_rate_compilation t2
+        ON ( t1.exchange_rate_date = t2.exchange_rate_date )
+        AND t2.provider = 'C2M'
+        AND t2.service_type = 'TBDF'
+
+    CROSS JOIN get_base_myr_value_for_comparison
+
+    WHERE 
+        t1.provider = 'CIEF'
+        AND
+        competitor_name IN ('C2M')
+        AND 
+        t1.service_type IN ('1 DAY TRANSFER')
+
+),
+
+restructure_and_expand_table_cief_3_days_vs_sd AS (
+    
+    SELECT
+        t1.provider,
+        t1.payment_method,
+        t1.service_type,
+        t1.myr_to_cny_currency_exchange_rate,
+        t1.exchange_rate_date,
+
+        t2.provider                 AS competitor_name,
+        t2.start_range_myr,
+        t2.end_range_myr,
+        t2.payment_method           AS competitor_payment_method,
+        t2.service_type             AS competitor_service_type,
+        t2.handling_fee_myr,
+        t2.handling_fee_cny,
+        ROUND(t2.myr_to_cny_currency_exchange_rate, 5)      AS competitor_myr_to_cny_currency_exchange_rate,
+        get_base_myr_value_for_comparison.comparison_base_value_myr                                
+
+    FROM currency_exchange_rate_compilation t1
+
+    LEFT JOIN currency_exchange_rate_compilation t2
+        ON ( t1.exchange_rate_date = t2.exchange_rate_date )
+        AND t2.provider = 'SD'
+    
+    CROSS JOIN get_base_myr_value_for_comparison
+
+    WHERE 
+        t1.provider = 'CIEF'
+        AND
+        competitor_name IN ('SD')
+        AND 
+        t1.service_type IN ('3 DAYS TRANSFER')
+
+),
+
+union_expanded_tables AS (
+    
+    SELECT * FROM restructure_and_expand_table_cief_vs_world_rate
+    UNION ALL
+    SELECT * FROM restructure_and_expand_table_cief_1688_vs_c2m_altr
+    UNION ALL
+    SELECT * FROM restructure_and_expand_table_cief_1_day_transfer_vs_c2m_tbdf
+    UNION ALL
+    SELECT * FROM restructure_and_expand_table_cief_3_days_vs_sd
+
+),
+
+rate_comparison AS (
+    
+    SELECT
+        *,
+        
+        ( (comparison_base_value_myr / 1.02) * myr_to_cny_currency_exchange_rate ) AS cief_amount_cny,
+
+        CASE
+            WHEN competitor_name = 'WORLD RATE'
+                THEN comparison_base_value_myr * competitor_myr_to_cny_currency_exchange_rate
+            WHEN competitor_name = 'C2M' AND competitor_service_type = 'ALTR'
+                THEN ( (comparison_base_value_myr - handling_fee_myr) * competitor_myr_to_cny_currency_exchange_rate )  
+            WHEN competitor_name = 'C2M' AND competitor_service_type = 'TBDF'
+                THEN comparison_base_value_myr * competitor_myr_to_cny_currency_exchange_rate
+            WHEN competitor_name = 'SD'
+                THEN ( comparison_base_value_myr * competitor_myr_to_cny_currency_exchange_rate ) - handling_fee_cny
+        END AS competitor_amount_cny,
+
+        MAX(exchange_rate_date) OVER ( PARTITION BY competitor_name, payment_method, competitor_payment_method)     AS latest_exchange_rate_date
+
+    FROM union_expanded_tables
+
+),
+
+order_metrics AS (
+
+    SELECT DISTINCT
+        service_type, 
+        order_payment_method,
+        DATE(order_created_datetime)     AS order_date,
+        SUM(order_quote_value) OVER (PARTITION BY DATE(order_created_datetime), service_type, order_payment_method)     AS daily_sales_cny,
+        SUM(order_quote_value) OVER (PARTITION BY DATE(order_created_datetime))                                         AS total_daily_sales_cny,
+
+        SUM(total_order_value_rm) OVER (PARTITION BY DATE(order_created_datetime), service_type, order_payment_method)  AS daily_sales_myr,
+        SUM(total_order_value_rm) OVER (PARTITION BY DATE(order_created_datetime))                                      AS total_daily_sales_myr
+
+    FROM orders
+
+    WHERE 
+        quote_currency_id = 2
+        AND
+        transaction_status = 'UPLOADED_BANK_SLIP'
+
+),
+
+exchange_rate_join_order_metrics AS (
+
+    SELECT
+        rate_comparison.provider,
+        rate_comparison.payment_method,
+        rate_comparison.service_type,
+        rate_comparison.myr_to_cny_currency_exchange_rate,
+        rate_comparison.exchange_rate_date,
+        rate_comparison.competitor_name,
+        rate_comparison.start_range_myr,
+        rate_comparison.end_range_myr,
+        rate_comparison.competitor_payment_method,
+        rate_comparison.competitor_service_type,
+        rate_comparison.handling_fee_myr,
+        rate_comparison.handling_fee_cny,
+        rate_comparison.competitor_myr_to_cny_currency_exchange_rate,
+        rate_comparison.comparison_base_value_myr,
+        rate_comparison.cief_amount_cny,
+        rate_comparison.competitor_amount_cny,
+        rate_comparison.latest_exchange_rate_date,
+        order_metrics.daily_sales_cny,
+        order_metrics.total_daily_sales_cny,
+        order_metrics.daily_sales_myr,
+        order_metrics.total_daily_sales_myr,
+
+        ROW_NUMBER() OVER (PARTITION BY exchange_rate_date ORDER BY COALESCE(total_daily_sales_cny,0) DESC)     AS partition_sort_idx,
+
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}'                       AS _dbt_ran_datetime 
+
+    FROM rate_comparison
+
+    LEFT JOIN order_metrics
+        ON 
+        ( 
+            rate_comparison.exchange_rate_date  = order_metrics.order_date 
+            AND rate_comparison.service_type    = order_metrics.service_type
+            AND rate_comparison.payment_method  = order_metrics.order_payment_method
+        )       
+
+),
+
+
+-- FINAL
+final__rep_exchange__exchange_rate_analysis AS (
+
+    SELECT
+
+        -- ids
+        partition_sort_idx,
+
+        -- dimensions
+        provider,
+        service_type,
+        payment_method,
+        competitor_name,
+        competitor_service_type,
+        competitor_payment_method,
+
+        -- measures
+        start_range_myr,
+        end_range_myr,
+        comparison_base_value_myr,
+        myr_to_cny_currency_exchange_rate,
+        competitor_myr_to_cny_currency_exchange_rate,
+        handling_fee_myr,
+        handling_fee_cny,
+        cief_amount_cny,
+        competitor_amount_cny,
+        daily_sales_cny,
+        total_daily_sales_cny,
+        daily_sales_myr,
+        total_daily_sales_myr,
+
+        -- date/times
+        exchange_rate_date,
+        latest_exchange_rate_date,
+
+        -- metadata
+        _dbt_ran_datetime
+
+    FROM exchange_rate_join_order_metrics
+
+)
+
+SELECT * FROM final__rep_exchange__exchange_rate_analysis

models/marts/warehouse/fct_exchange__currency_exchange_rate_compilation.sql

@@ -0,0 +1,234 @@
+-- IMPORT
+WITH open_api_exchange_rate_logs AS (
+    SELECT * FROM {{ ref('stg_open_ex_api__open_api_currency_rate_logs') }}
+),
+
+cief_exchange_rate_logs AS (
+    SELECT * FROM {{ ref('stg_exchange__currency_rate_logs') }}
+),
+
+c2m_exchange_rate_logs AS (
+    SELECT * FROM {{ ref('stg_c2m_exchange__c2m_currency_rate_logs') }}
+),
+
+sd_exchange_rate_logs AS (
+    SELECT * FROM {{ ref('stg_googlesheet__sd_currency_exchange_rates') }}
+),
+
+
+-- LOGIC
+-- CIEF Rates
+cief_exchange_rate_log_filtering AS (
+
+    SELECT 
+        *
+
+    FROM cief_exchange_rate_logs
+
+    WHERE 
+        payment_method NOT IN ('BA') 
+        AND
+        quote_currency_id = 2
+
+),
+
+cief_exchange_rates AS (
+
+    SELECT
+        'CIEF'              AS provider,
+        1                   AS start_range_myr,
+        200000              AS end_range_myr,
+        payment_method,
+        service_type,
+        NULL                AS handling_fee_myr,
+        NULL                AS handling_fee_cny,
+        base_to_quote_currency_exchange_rate    AS myr_to_cny_currency_exchange_rate,
+
+        open_api_exchange_rate_logs.exchange_rate_date
+
+    FROM cief_exchange_rate_log_filtering
+
+    INNER JOIN open_api_exchange_rate_logs
+        ON ( open_api_exchange_rate_logs.exchange_rate_date >= cief_exchange_rate_log_filtering.from_datetime )
+            AND (open_api_exchange_rate_logs.exchange_rate_date < cief_exchange_rate_log_filtering.to_datetime) 
+
+),
+
+-- World Rates
+world_rate_exchange_rates AS (
+
+    SELECT
+        'WORLD RATE'        AS provider,
+        1                   AS start_range_myr,
+        200000              AS end_range_myr,
+        NULL                AS payment_method,
+        NULL                AS service_type,
+        NULL                AS handling_fee_myr,
+        NULL                AS handling_fee_cny,
+        myr_to_cny_rate     AS myr_to_cny_currency_exchange_rate,
+        exchange_rate_date
+
+    FROM open_api_exchange_rate_logs
+
+),
+
+-- C2M Rates
+c2m_ranked_values AS (
+
+    SELECT
+        base_value_myr,
+        myr_to_cny_currency_exchange_rate,
+        handling_fee_myr,
+        created_datetime,
+        service_type,
+        LAG(base_value_myr, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr ) AS prev_base_value_myr,
+        LEAD(base_value_myr, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr ) AS next_base_value_myr,
+
+        LAG(myr_to_cny_currency_exchange_rate, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr)    AS prev_rate,
+        LEAD(myr_to_cny_currency_exchange_rate, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr)   AS next_rate,
+        
+        LAG(handling_fee_myr, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr)     AS prev_fee,
+        LEAD(handling_fee_myr, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr)    AS next_fee
+
+    FROM c2m_exchange_rate_logs
+
+),
+
+c2m_rate_changed_rows AS (
+
+    SELECT
+        base_value_myr,
+        myr_to_cny_currency_exchange_rate,
+        handling_fee_myr,
+        created_datetime,
+        service_type,
+
+        CASE
+            WHEN LAG(myr_to_cny_currency_exchange_rate) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr) != myr_to_cny_currency_exchange_rate
+                OR LAG(handling_fee_myr) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr) != handling_fee_myr
+            THEN 1
+            ELSE 0
+        END AS is_rate_change_row
+
+    FROM c2m_exchange_rate_logs
+
+),
+
+c2m_group_changed_rows AS (
+
+    SELECT
+        base_value_myr,
+        myr_to_cny_currency_exchange_rate,
+        handling_fee_myr,
+        created_datetime,
+        service_type,
+        SUM(is_rate_change_row) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr)          AS group_id,
+        LEAD(base_value_myr, 1) OVER (PARTITION BY created_datetime, service_type ORDER BY created_datetime, service_type, base_value_myr ) - 1     AS next_base_value_myr
+
+    FROM c2m_rate_changed_rows
+    
+),
+
+c2m_base_value_ranges AS (
+
+    SELECT
+        'C2M'                                           AS provider,
+        MIN(base_value_myr)                             AS start_range_myr,
+        COALESCE( MAX(next_base_value_myr), 200000 )    AS end_range_myr,
+        NULL                                            AS payment_method,
+        service_type,
+        handling_fee_myr,
+        NULL                                            AS handling_fee_cny,
+        myr_to_cny_currency_exchange_rate,          
+        DATE(created_datetime)                          AS exchange_rate_date
+
+    FROM c2m_group_changed_rows
+
+    GROUP BY 
+        service_type,
+        myr_to_cny_currency_exchange_rate, 
+        handling_fee_myr,
+        DATE(created_datetime),
+        group_id
+        
+),
+
+-- SD Group Rates
+sd_exchange_rates AS (
+
+    SELECT
+        'SD'                AS provider,
+        1                   AS start_range_myr,
+        200000              AS end_range_myr,
+        payment_method, 
+        NULL                AS service_type,
+        NULL                AS handling_fee_myr,
+        handling_fee_cny,
+        myr_to_cny_currency_exchange_rate,
+        exchange_rate_date
+
+    FROM sd_exchange_rate_logs
+
+),
+
+union_competitor_rates AS (
+    
+    SELECT * FROM world_rate_exchange_rates
+
+    UNION ALL
+
+    SELECT * FROM cief_exchange_rates
+
+    UNION ALL
+
+    SELECT * FROM c2m_base_value_ranges 
+    
+    UNION ALL 
+    
+    SELECT * FROM sd_exchange_rates
+
+),
+
+add_metadata AS (
+    
+    SELECT
+        *,
+
+        MD5_NUMBER_LOWER64( CONCAT(provider, myr_to_cny_currency_exchange_rate, exchange_rate_date) )       AS exchange_rate_log_id,
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}'                   AS _dbt_ran_datetime
+
+    FROM union_competitor_rates
+
+),
+
+
+-- FINAL
+final__fct_exchange__currency_exchange_rate_compilation AS (
+
+    SELECT
+        --id
+        exchange_rate_log_id,
+
+        --dimension
+        provider,
+        payment_method,
+        service_type,
+
+        --measures
+        start_range_myr,
+        end_range_myr,
+        handling_fee_myr,
+        handling_fee_cny,
+        myr_to_cny_currency_exchange_rate,
+        
+        --date/time
+        exchange_rate_date,
+
+        --metadata
+        _dbt_ran_datetime
+
+    FROM add_metadata
+
+)
+
+SELECT * FROM final__fct_exchange__currency_exchange_rate_compilation

models/staging/currency_exchange_rates/c2m/_c2m_webscrape_exchange_rate__sources.yml

@@ -0,0 +1,29 @@
+version: 2
+
+sources:
+  - name: src_c2m_exchange_rates
+    description: Currency exchange rate data of competitor C2M, obtained from web scraping. Link=https://www.c2m.my/
+    database: DEV_CIEF_RAW_DB
+    schema: C2M_WEBSCRAPE
+    loader: custom_python_code
+    loaded_at_field: SCRAPE_DATE
+    freshness:
+      warn_after: {count: 26, period: hour}
+      error_after: {count: 48, period: hour}
+    meta: 
+      owner: "@yam"
+      model_maturity: prod
+    tags:
+      - exchange_rate_log
+      - daily 
+
+
+    tables: 
+      - name: currency_rate_logs
+        description: Web scraped daily currency exchange rates of different products offered by competitor C2M.
+        columns:
+          - name: service_type
+            description: (
+              ALTR:ALIPAY:Transfer/Scan-To-Pay, 
+              TBDF:TAOBAO/1688:Payment-On-Behalf
+              )

models/staging/currency_exchange_rates/c2m/stg_c2m_exchange__c2m_currency_rate_logs.sql

@@ -0,0 +1,114 @@
+-- IMPORT
+WITH c2m_exchange_rate_log AS (
+    SELECT * FROM {{ source('src_c2m_exchange_rates', 'currency_rate_logs') }}
+),
+
+
+-- LOGIC
+data_cleaning AS (
+    SELECT
+        *,
+
+        -- remove handling fee for TBDF
+        CASE 
+            WHEN service_type = 'TBDF'
+                THEN 0
+                ELSE handeling_fee
+        END AS cleaned_handling_fee
+
+    FROM c2m_exchange_rate_log
+),
+
+rename_and_casting AS (
+
+    SELECT
+        product_name,
+        site_name                           AS competitor_name,
+        service_type,
+        pending_order                       AS order_queue_number,
+        myr::FLOAT                          AS base_value_myr,
+        cny::FLOAT                          AS quote_value_cny,
+        cleaned_handling_fee::FLOAT         AS handling_fee_myr,
+        REPLACE(payable, ',','')::FLOAT     AS payable_myr,
+        current_rate::FLOAT                 AS myr_to_cny_currency_exchange_rate,
+        staff_status,
+        DATEADD(DAY, -1, scrape_date)       AS created_datetime,
+        scrape_hour,
+
+        TO_TIMESTAMP(
+                REGEXP_SUBSTR(
+                    last_rate_update, 
+                    '[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{1,2}:[0-9]{2} [APap][Mm]'
+                ), 
+            'YYYY-MM-DD HH12:MI AM'
+        )       AS rate_updated_datetime
+
+    FROM data_cleaning
+
+),
+
+recalculate_payable_column AS (
+    
+    SELECT
+        MD5_NUMBER_LOWER64(CONCAT(service_type, base_value_myr, created_datetime ))             AS currency_rate_log_id,
+
+        product_name,
+        competitor_name,
+        service_type,
+        order_queue_number,
+        base_value_myr,
+        quote_value_cny,
+        handling_fee_myr,
+        myr_to_cny_currency_exchange_rate,
+        staff_status,
+        created_datetime,
+        scrape_hour,
+        rate_updated_datetime,
+
+        -- Recalculate payable myr after removing handling fee
+        CASE 
+            WHEN service_type = 'TBDF'
+                THEN base_value_myr
+                ELSE payable_myr
+        END     AS payable_myr,
+
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}'       AS _dbt_ran_datetime
+
+    FROM rename_and_casting
+
+),
+
+
+-- FINAL
+final__stg_exchange__c2m_exchange_rates AS (
+
+    SELECT
+        -- ids
+        currency_rate_log_id,
+
+        -- dimensions
+        competitor_name,
+        product_name,
+        service_type,
+        staff_status,
+
+        -- measures
+        order_queue_number,
+        base_value_myr,
+        quote_value_cny,
+        myr_to_cny_currency_exchange_rate,
+        handling_fee_myr,
+        payable_myr,
+
+        -- date/time
+        rate_updated_datetime,
+        created_datetime,
+
+        -- metadata
+        _dbt_ran_datetime
+    
+    FROM recalculate_payable_column
+
+)
+
+SELECT * FROM final__stg_exchange__c2m_exchange_rates

models/staging/currency_exchange_rates/world_rate/_open_api_exchange_rate__sources.yml

@@ -0,0 +1,29 @@
+version: 2
+
+sources:
+  - name: src_open_api_exchange_rates
+    description: Latest currency exchange rates of multiple currencies obtained from `Open Exchange Rates` API call.
+    database: DEV_CIEF_RAW_DB
+    schema: OPENEXCHANGERATES_AIRBYTE
+    loader: custom_python_code
+    loaded_at_field: _airbyte_extracted_at
+    freshness:
+      warn_after: {count: 26, period: hour}
+      error_after: {count: 48, period: hour}
+    meta: 
+      owner: "@yam"
+      model_maturity: prod
+    tags:
+      - exchange_rate_log
+      - daily 
+
+
+    tables: 
+      - name: current_rate
+        description: Hourly currency exchange rates obtained from the API call.
+        columns:
+          - name: _airbyte_raw_id
+            description: Primary key for 'current_rate'
+            tests:
+              - unique
+              - not_null

models/staging/currency_exchange_rates/world_rate/stg_open_ex_api__open_api_currency_rate_logs.sql

@@ -0,0 +1,108 @@
+-- IMPORT
+WITH api_call_exchange_rates AS (
+    SELECT * FROM {{ source('src_open_api_exchange_rates', 'current_rate') }}
+),
+
+seed_historical_exchange_rates AS (
+    SELECT * FROM {{ ref('seed_exchange__historical_currency_exchange_rates') }}
+),
+
+
+-- LOGIC
+api_rename_and_casting AS (
+
+    SELECT
+        _airbyte_raw_id                         AS _airbyte_id,
+        base                                    AS base_currency,
+        PARSE_JSON(rates::STRING)               AS parsed_rates,
+        _airbyte_extracted_at                   AS _airbyte_extracted_datetime,
+        TO_TIMESTAMP_NTZ(timestamp::INT)        AS exchange_rate_datetime
+
+    FROM api_call_exchange_rates
+
+),
+
+api_currency_conversion AS (
+
+    SELECT
+        DATE(exchange_rate_datetime)                        AS exchange_rate_date,
+        AVG(1 / parsed_rates['MYR'])                        AS myr_to_usd_rate,
+        AVG(parsed_rates['CNY'] / parsed_rates['MYR'])      AS myr_to_cny_rate
+        
+    FROM api_rename_and_casting
+
+    GROUP BY 
+        DATE(exchange_rate_datetime)
+
+),
+
+seed_currency_conversion AS (
+
+    SELECT
+        TO_CHAR(TO_DATE(date,'DD-MM-YYYY'), 'YYYY-MM-DD')            AS exchange_rate_date,
+        cny / myr       AS myr_to_cny_rate,
+        1 / myr         AS myr_to_usd_rate
+
+    FROM seed_historical_exchange_rates
+
+),
+
+union_exchange_rates AS (
+
+    SELECT 
+        exchange_rate_date,
+        myr_to_cny_rate,
+        myr_to_usd_rate
+        
+    FROM seed_currency_conversion
+
+    WHERE 
+        exchange_rate_date < '2024-01-04'
+
+    UNION ALL 
+
+    SELECT
+        exchange_rate_date,
+        myr_to_cny_rate,
+        myr_to_usd_rate
+
+    FROM api_currency_conversion
+
+),
+
+add_primary_key_column AS (
+
+    SELECT
+        *,
+        MD5_NUMBER_LOWER64(CONCAT(myr_to_cny_rate, myr_to_usd_rate, exchange_rate_date))        AS currency_rate_log_id,
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}'       AS _dbt_ran_datetime
+    
+    FROM union_exchange_rates
+
+),
+
+
+-- FINAL
+final__stg_exchange__api_call_exchange_rates AS (
+
+    SELECT
+        -- id
+        currency_rate_log_id,
+
+        -- dimension
+        
+        -- measure
+        myr_to_usd_rate,
+        myr_to_cny_rate,
+
+        -- date/time
+        exchange_rate_date,
+
+        -- metadata
+        _dbt_ran_datetime
+
+    FROM add_primary_key_column
+
+)
+
+SELECT * FROM final__stg_exchange__api_call_exchange_rates

models/staging/exchange/stg_exchange__currency_rate_logs.sql

@@ -41,9 +41,9 @@ currency_rate_logs_join_currency_rates AS (
 
     LEFT JOIN currency_rates
         ON (currency_rate_logs.currency_rate_id = currency_rates.currency_rate_id)
+        
 ),
 
-
 currency_rate_logs_quote_currency_id_join_currencies AS (
 
     SELECT
@@ -58,7 +58,7 @@ currency_rate_logs_quote_currency_id_join_currencies AS (
         currency_rate_logs_join_currency_rates.base_to_quote_currency_exchange_rate,
 
         COALESCE(payment_methods.name, currency_rate_logs_join_currency_rates.payment_method::string)   AS payment_method,
-        COALESCE(service_types.name, currency_rate_logs_join_currency_rates.service_type::string)   AS service_type,
+        COALESCE(service_types.name, currency_rate_logs_join_currency_rates.service_type::string)       AS service_type,
 
         currency_rate_logs_join_currency_rates.creator_user_id,
         currency_rate_logs_join_currency_rates.created_datetime,
@@ -75,66 +75,81 @@ currency_rate_logs_quote_currency_id_join_currencies AS (
 
     LEFT JOIN service_types
         ON (currency_rate_logs_join_currency_rates.service_type = service_types.service_type_id)
+
+),
+
+filtered_currency_rate_logs AS (
+
+    SELECT
+        *
+
+    FROM currency_rate_logs_quote_currency_id_join_currencies
+
+    WHERE service_type NOT IN ('x1', 'Company to Company', 'RECEIVED-ON-BEHALF')
+
 ),
 
 currency_rate_logs_add_dense_rank AS ( 
 
             SELECT  
-                currency_rate_logs_quote_currency_id_join_currencies.currency_rate_log_id,
+                filtered_currency_rate_logs.currency_rate_log_id,
 
-                dense_rank() OVER 
+                DENSE_RANK() OVER 
-                    (PARTITION BY currency_rate_logs_quote_currency_id_join_currencies.currency_rate_id 
+                    (PARTITION BY filtered_currency_rate_logs.currency_rate_id 
-                    ORDER BY currency_rate_logs_quote_currency_id_join_currencies.created_datetime)                 AS dense_rank_index,
+                        ORDER BY filtered_currency_rate_logs.created_datetime)          AS dense_rank_index,
 
-                currency_rate_logs_quote_currency_id_join_currencies.currency_rate_id                         AS currency_rate_id,
+                filtered_currency_rate_logs.currency_rate_id                            AS currency_rate_id,
-                currency_rate_logs_quote_currency_id_join_currencies.quote_currency_id,
+                filtered_currency_rate_logs.quote_currency_id,
-                currency_rate_logs_quote_currency_id_join_currencies.quote_currency_name,
+                filtered_currency_rate_logs.quote_currency_name,
-                currency_rate_logs_quote_currency_id_join_currencies.quote_currency_short_code,
+                filtered_currency_rate_logs.quote_currency_short_code,
-                currency_rate_logs_quote_currency_id_join_currencies.quote_currency_symbol,
+                filtered_currency_rate_logs.quote_currency_symbol,
-                currency_rate_logs_quote_currency_id_join_currencies.payment_method,
+                filtered_currency_rate_logs.payment_method,
-                currency_rate_logs_quote_currency_id_join_currencies.service_type,
+                filtered_currency_rate_logs.service_type,
-                currency_rate_logs_quote_currency_id_join_currencies.base_to_quote_currency_exchange_rate,
+                filtered_currency_rate_logs.base_to_quote_currency_exchange_rate,
-                currency_rate_logs_quote_currency_id_join_currencies.creator_user_id,
+                filtered_currency_rate_logs.creator_user_id,
-                currency_rate_logs_quote_currency_id_join_currencies.created_datetime
+                filtered_currency_rate_logs.created_datetime
 
-            FROM currency_rate_logs_quote_currency_id_join_currencies
+            FROM filtered_currency_rate_logs
 
             ORDER BY 
                 currency_rate_id, 
                 created_datetime
+
 ),
 
+currency_rate_logs_resturucure AS (   
 
-currency_rate_logs_resturucure AS (             
+    SELECT  
-            SELECT  
+        currency_rate_logs_add_dense_rank.currency_rate_log_id,
-                currency_rate_logs_add_dense_rank.currency_rate_log_id,
+        currency_rate_logs_add_dense_rank.currency_rate_id,
-                currency_rate_logs_add_dense_rank.currency_rate_id,
+        currency_rate_logs_add_dense_rank.quote_currency_id,
-                currency_rate_logs_add_dense_rank.quote_currency_id,
+        currency_rate_logs_add_dense_rank.quote_currency_name,
-                currency_rate_logs_add_dense_rank.quote_currency_name,
+        currency_rate_logs_add_dense_rank.quote_currency_short_code,
-                currency_rate_logs_add_dense_rank.quote_currency_short_code,
+        currency_rate_logs_add_dense_rank.quote_currency_symbol,
-                currency_rate_logs_add_dense_rank.quote_currency_symbol,
+        currency_rate_logs_add_dense_rank.payment_method,
-                currency_rate_logs_add_dense_rank.payment_method,
+        currency_rate_logs_add_dense_rank.service_type,
-                currency_rate_logs_add_dense_rank.service_type,
+        currency_rate_logs_add_dense_rank.base_to_quote_currency_exchange_rate,
-                currency_rate_logs_add_dense_rank.base_to_quote_currency_exchange_rate,
+        currency_rate_logs_add_dense_rank.creator_user_id,
-                currency_rate_logs_add_dense_rank.creator_user_id,
 
-                CASE
+        CASE
-                    WHEN currency_rate_logs_add_dense_rank.dense_rank_index=1 THEN CAST('1990-01-01 00:00:00 +08:00' AS DATETIME) 
+            WHEN currency_rate_logs_add_dense_rank.dense_rank_index=1 THEN CAST('1990-01-01 00:00:00 +08:00' AS DATETIME) 
-                    ELSE currency_rate_logs_add_dense_rank.created_datetime
+            ELSE currency_rate_logs_add_dense_rank.created_datetime
-                END                                                                             AS from_datetime,
+        END                                                                             AS from_datetime,
 
-                COALESCE(currency_rate_logs_add_dense_rank_clone.created_datetime, 
+        COALESCE(currency_rate_logs_add_dense_rank_clone.created_datetime, 
-                        CAST('2999-12-31 23:59:59 +08:00' AS DATETIME))                         AS to_datetime,
+                CAST('2999-12-31 23:59:59 +08:00' AS DATETIME))                         AS to_datetime,
-                '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
 
-            FROM    
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
-                currency_rate_logs_add_dense_rank 
 
-            LEFT JOIN currency_rate_logs_add_dense_rank AS currency_rate_logs_add_dense_rank_clone
+    FROM    
-                ON currency_rate_logs_add_dense_rank.dense_rank_index = currency_rate_logs_add_dense_rank_clone.dense_rank_index-1 
+        currency_rate_logs_add_dense_rank 
-                    AND currency_rate_logs_add_dense_rank.currency_rate_id = currency_rate_logs_add_dense_rank_clone.currency_rate_id 
+
+    LEFT JOIN currency_rate_logs_add_dense_rank AS currency_rate_logs_add_dense_rank_clone
+        ON currency_rate_logs_add_dense_rank.dense_rank_index = currency_rate_logs_add_dense_rank_clone.dense_rank_index-1 
+            AND currency_rate_logs_add_dense_rank.currency_rate_id = currency_rate_logs_add_dense_rank_clone.currency_rate_id 
+
+    ORDER BY currency_rate_logs_add_dense_rank.currency_rate_id, currency_rate_logs_add_dense_rank.created_datetime
 
-            ORDER BY currency_rate_logs_add_dense_rank.currency_rate_id, currency_rate_logs_add_dense_rank.created_datetime
 ), 
 
 

models/staging/googlesheet/_googlesheet__sources.yml

@@ -44,6 +44,16 @@ sources:
         columns:
           - name: _airbyte_raw_id
             description: Primary key for 'account_frozen_case'
+            tests: 
+              - unique
+              - not_null
+
+              
+      - name: sd_currency_exchange_rates
+        description: Data retrieved from gsheet, comprises the currency exchange rates from our competitor SD.
+        columns:
+          - name: _airbyte_raw_id
+            description: Primary key for 'sd_currency_exchange_rates'
             tests: 
               - unique
               - not_null

models/staging/googlesheet/stg_googlesheet__sd_currency_exchange_rates.sql

@@ -0,0 +1,69 @@
+-- IMPORT
+WITH sd_exchange_rate_log AS (
+    SELECT * FROM {{ source('src_googlesheet_airbyte', 'sd_currency_exchange_rates') }}
+),
+
+seed_metadata AS (
+    SELECT * FROM {{ ref('seed_exchange__metadata') }}
+),
+
+
+-- LOGIC
+rename_and_casting AS (
+
+    SELECT
+        {{ dbt_utils.generate_surrogate_key(['payment_method', 'myr_to_cny_rate', 'exchange_rate_date']) }} AS sd_exchange_rate_id,
+
+        _airbyte_raw_id                                 AS _airbyte_id,
+        
+        sd_exchange_rate_log.provider                   AS competitor_name,
+        sd_exchange_rate_log.payment_method,
+        sd_exchange_rate_log.myr_to_cny_rate::FLOAT     AS myr_to_cny_currency_exchange_rate,  
+        sd_exchange_rate_log.transfer_days              AS transfer_duration_remark,
+        sd_exchange_rate_log.remark,
+
+        COALESCE(    
+            TRY_TO_DATE(sd_exchange_rate_log.exchange_rate_date, 'DD-MM-YY'),
+            TRY_TO_DATE(sd_exchange_rate_log.exchange_rate_date, 'MM-DD-YY')
+        )   AS exchange_rate_date,
+
+        seed_metadata.value                         AS handling_fee_cny,
+
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}'       AS _dbt_ran_datetime
+
+    FROM sd_exchange_rate_log
+
+    LEFT JOIN seed_metadata
+        ON (seed_metadata.table_name = 'stg_exchange__sd_currency_rate_logs')
+
+),
+
+
+-- FINAL
+final__stg_exchange__sd_currency_rate_logs AS (
+
+    SELECT
+        -- ids
+        sd_exchange_rate_id,
+
+        -- dimensions
+        competitor_name,
+        payment_method,
+        transfer_duration_remark,
+        remark,
+
+        -- measures
+        handling_fee_cny,
+        myr_to_cny_currency_exchange_rate,
+
+        -- date/time
+        exchange_rate_date,
+
+        -- metadata
+        _dbt_ran_datetime
+    
+    FROM rename_and_casting
+
+)
+
+SELECT * FROM final__stg_exchange__sd_currency_rate_logs

seeds/exchange/seed_exchange__metadata.csv

@@ -0,0 +1,2 @@
+ID,TABLE_NAME,SETTING_NAME,COLUMN_NAME,LOGIC,VALUE,LAST_UPDATED_DATE
+1,stg_exchange__sd_currency_rate_logs,sd have 20CNY charge every transaction,handling_fee_cny,add column,20,2024-02-20