cief/dbt_cloud
1
0
Fork 0
mirror of https://gitlab.com/cief-data/dbt_cloud.git synced 2026-08-19 04:14:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'lee_build_int_get_latest_cost' into 'main'

Browse Source 
Redefine Status and build more REP

See merge request cief-data/dbt_cloud!27
This commit is contained in:
CIEF ACC1
2023-10-10 06:56:27 +00:00
parent 23aa258e68 d59b6908ef
commit ff56b3c02e
11 changed files with 1109 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql
+158
View File
@@ -0,0 +1,158 @@
-- VARIABLES
{% set order_status = dbt_utils.get_column_values(
table=ref('stg_exchange__transaction_order_logs'),
column='status')
%}
-- IMPORTS
WITH transaction_cost_logs AS (
SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }}
),
transaction_costs AS (
SELECT * FROM {{ ref('stg_exchange__transaction_costs') }}
),
-- LOGIC
transaction_cost_status_datetime AS (
SELECT
transaction_cost_id,
company_id,
MIN(created_datetime) AS cost_created_datetime,
{%- for status in order_status %}
MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%}
{% endfor %}
FROM transaction_cost_logs
GROUP BY
transaction_cost_id,
company_id
),
join_cost_and_logs AS (
SELECT
transaction_costs.transaction_cost_id,
transaction_costs.transaction_order_id,
transaction_costs.supplier_company_id,
transaction_costs.bank_id,
transaction_costs.base_currency_id,
transaction_costs.quote_currency_id,
transaction_costs.transaction_type,
transaction_costs.payment_method,
transaction_costs.status,
transaction_costs.payment_reference,
transaction_costs.bill_number,
transaction_costs.base_value,
transaction_costs.quote_value,
transaction_costs.base_to_quote_currency_exchange_rate,
transaction_costs.base_tax,
transaction_costs.base_service_charge,
transaction_costs.expired_datetime,
transaction_costs.deleted_datetime,
transaction_costs.created_datetime,
transaction_costs.updated_datetime,
transaction_cost_status_datetime.company_id,
transaction_cost_status_datetime.cost_created_datetime,
transaction_cost_status_datetime.cost_completed_datetime,
transaction_cost_status_datetime.cost_pending_submission_datetime,
transaction_cost_status_datetime.cost_approved_datetime,
transaction_cost_status_datetime.cost_pending_verification_datetime,
transaction_cost_status_datetime.cost_rejected_datetime,
transaction_cost_status_datetime.cost_suspended_datetime,
transaction_cost_status_datetime.cost_expired_datetime
FROM transaction_costs
LEFT JOIN transaction_cost_status_datetime
ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id)
),
remove_deleted_expired_costs AS (
SELECT
*
FROM join_cost_and_logs
WHERE
deleted_datetime IS NULL
AND
expired_datetime IS NULL
),
-- Omair mentioned a bug in system causing duplicates. Take cost with latest updated_datetime as true.
remove_system_error_duplicate_row AS (
SELECT
*,
ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM remove_deleted_expired_costs
QUALIFY
row_number_index = 1
),
-- FINAL
final__int_exchange__transaction_cost_get_latest_cost_ids AS (
SELECT
-- ids
transaction_cost_id,
transaction_order_id,
supplier_company_id,
company_id,
bank_id,
base_currency_id,
quote_currency_id,
-- dimensions
transaction_type,
payment_method,
status,
payment_reference,
bill_number,
-- measures
base_value,
quote_value,
base_to_quote_currency_exchange_rate,
base_tax,
base_service_charge,
-- date/times
expired_datetime,
deleted_datetime,
created_datetime,
updated_datetime,
cost_created_datetime,
cost_completed_datetime,
cost_pending_submission_datetime,
cost_approved_datetime,
cost_pending_verification_datetime,
cost_rejected_datetime,
cost_suspended_datetime,
cost_expired_datetime,
-- metadata
_dbt_ran_datetime
FROM remove_system_error_duplicate_row
)
SELECT * FROM final__int_exchange__transaction_cost_get_latest_cost_ids
models/marts/reporting/rep_exchange__cohort_analysis.sql
+187
View File
@@ -0,0 +1,187 @@
-- SOURCE: https://www.holistics.io/blog/calculate-cohort-retention-analysis-with-sql/
-- IMPORT
WITH companies AS (
SELECT * FROM {{ ref('dim_exchange__companies') }}
),
transaction_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
-- LOGIC
company_register_month_year AS (
SELECT
companies.company_id,
companies.company_created_datetime,
IFF(MAX(transaction_orders.order_created_datetime) IS NULL, 1, 0) AS is_never_order_company,
DATE(DATE_TRUNC('MONTH', companies.company_created_datetime)) AS register_month_year,
DATE(DATE_TRUNC('MONTH', MIN(transaction_orders.order_created_datetime))) AS first_order_month_year
FROM companies
LEFT JOIN transaction_orders
ON (companies.company_id = transaction_orders.company_id)
GROUP BY
companies.company_id,
companies.company_created_datetime,
register_month_year
),
-- Identify orders placed from each company, from the time of company registration
-- In cohort analysis, we do not consider number of orders by each user, just whether has_placed_orders or not_placed_orders in each month
company_order_count_from_registration_date AS (
SELECT
transaction_orders.company_id,
DATEDIFF(MONTH, company_register_month_year.company_created_datetime, transaction_orders.order_created_datetime) AS order_month
FROM transaction_orders
LEFT JOIN company_register_month_year
ON (transaction_orders.company_id = company_register_month_year.company_id)
GROUP BY
transaction_orders.company_id,
order_month
),
company_first_order_count_from_registration_date AS (
SELECT
transaction_orders.company_id,
DATE(DATE_TRUNC('MONTH', company_created_datetime)) AS company_created_datetime,
MIN(DATEDIFF(MONTH, company_register_month_year.company_created_datetime, transaction_orders.order_created_datetime)) AS first_order_month
FROM transaction_orders
LEFT JOIN company_register_month_year
ON (transaction_orders.company_id = company_register_month_year.company_id)
GROUP BY
transaction_orders.company_id,
company_created_datetime
),
cohort_size_by_month_year AS (
SELECT
register_month_year,
SUM(is_never_order_company) AS count_never_order_companies,
COUNT(register_month_year) AS count_total_companies
FROM company_register_month_year
GROUP BY register_month_year
ORDER BY register_month_year
),
order_retention AS (
SELECT
company_register_month_year.register_month_year,
company_order_count_from_registration_date.order_month,
COUNT(company_register_month_year.register_month_year) AS count_retained_companies
FROM company_order_count_from_registration_date
LEFT JOIN company_register_month_year
ON (company_order_count_from_registration_date.company_id = company_register_month_year.company_id)
GROUP BY
company_register_month_year.register_month_year,
company_order_count_from_registration_date.order_month
),
first_time_order_retention AS (
SELECT
company_created_datetime,
first_order_month,
COUNT(first_order_month) AS count_first_order_companies
FROM company_first_order_count_from_registration_date
GROUP BY
company_created_datetime,
first_order_month
),
cohort_analysis_table AS (
SELECT
order_retention.register_month_year,
order_retention.order_month,
order_retention.count_retained_companies AS retained_companies,
cohort_size_by_month_year.count_total_companies AS total_registered_company,
cohort_size_by_month_year.count_never_order_companies,
first_time_order_retention.count_first_order_companies AS count_first_order_companies,
DIV0(
retained_companies,
total_registered_company
) AS retention_rate_overall,
DIV0(
retained_companies,
( total_registered_company - cohort_size_by_month_year.count_never_order_companies )
) AS retention_rate_exclude_never_order_company,
DIV0(
count_first_order_companies,
( total_registered_company - count_never_order_companies )
) AS first_order_percentage,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM order_retention
LEFT JOIN cohort_size_by_month_year
ON (order_retention.register_month_year = cohort_size_by_month_year.register_month_year)
LEFT JOIN first_time_order_retention
ON (order_retention.register_month_year = first_time_order_retention.company_created_datetime)
AND (order_retention.order_month = first_time_order_retention.first_order_month)
ORDER BY
order_retention.register_month_year,
order_retention.order_month
),
-- FINAL
final_rep_exchange__cohort_analysis AS (
SELECT
-- dimensions
register_month_year,
total_registered_company,
order_month,
-- measures
count_never_order_companies,
count_first_order_companies,
retained_companies,
retention_rate_overall,
retention_rate_exclude_never_order_company,
first_order_percentage,
-- metadata
_dbt_ran_datetime
FROM cohort_analysis_table
)
SELECT * FROM final_rep_exchange__cohort_analysis
models/marts/reporting/rep_exchange__daily_orders.sql
+83 -35
View File
@@ -1,3 +1,23 @@
-- VARIABLES
{% set var_service_type = [
'1688 PAYMENT',
'1 DAY TRANSFER',
'3 DAYS TRANSFER',
'Enterprise to Enterprise 公打公'] %}
{% set var_working_days_including_base_date = [
'add_1_working_day_included_base_date',
'add_1_working_day_included_base_date',
'add_3_working_day_included_base_date',
'add_7_working_day_included_base_date'] %}
{% set var_working_days_excluding_base_date = [
'add_1_working_day_excluded_base_date',
'add_1_working_day_excluded_base_date',
'add_3_working_day_excluded_base_date',
'add_7_working_day_excluded_base_date'] %}
-- IMPORT
WITH orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
@@ -25,6 +45,8 @@ fct_and_dim_joins AS (
orders.booking_id,
orders.booking_marking_id,
orders.company_id,
orders.transaction_status,
orders.document_status,
companies.name AS company_name,
companies.company_marking_id AS company_marking_id,
@@ -40,8 +62,9 @@ fct_and_dim_joins AS (
companies.has_wallet AS company_has_wallet,
companies.latitude AS company_latitude,
companies.longitude AS company_longitude,
companies.company_created_datetime,
companies.company_lifetime_value,
companies.m_score_lifetime,
companies.m_score_lifetime AS company_m_score_lifetime,
orders.user_id,
orders.bank_id,
@@ -93,63 +116,56 @@ fct_and_dim_joins AS (
-- Estimated delivery datetime based on CIEF internal SLA (cutoff time 4pm)
CASE
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1 DAY TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_included_base_date || ' 23:59:59 +08:00')
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1 DAY TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_excluded_base_date || ' 23:59:59 +08:00')
{% for (loop_service_type, working_day_with_base_date, working_day_without_base_date) in zip(var_service_type, var_working_days_including_base_date, var_working_days_excluding_base_date) %}
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '3 DAYS TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.add_3_working_day_included_base_date || ' 23:59:59 +08:00')
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '3 DAYS TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.add_3_working_day_excluded_base_date || ' 23:59:59 +08:00')
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day_with_base_date}} || ' 23:59:59 +08:00')
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day_without_base_date}} || ' 23:59:59 +08:00')
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1688 PAYMENT' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_included_base_date || ' 23:59:59 +08:00')
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1688 PAYMENT' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_excluded_base_date || ' 23:59:59 +08:00')
{% endfor %}
ELSE NULL
ELSE null
END AS estimated_order_delivery_datetime_website_sla,
--- Estimated delivery datetime based on customer expectation
CASE
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1 DAY TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_included_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1 DAY TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_excluded_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
{% for (loop_service_type, working_day_with_base_date, working_day_without_base_date) in zip(var_service_type, var_working_days_including_base_date, var_working_days_excluding_base_date) %}
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '3 DAYS TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.add_3_working_day_included_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '3 DAYS TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.add_3_working_day_excluded_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day_with_base_date}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day_without_base_date}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1688 PAYMENT' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_included_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1688 PAYMENT' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_excluded_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
{% endfor %}
ELSE NULL
ELSE null
END AS estimated_order_delivery_datetime_customer_expectation,
-- On time delivery boolean (CIEF internal SLA)
CASE
WHEN order_completed_datetime IS NULL THEN NULL
WHEN estimated_order_delivery_datetime_website_sla > order_completed_datetime THEN 1
-- On-time boolean is null when order is not complete
WHEN account_verified_payment_datetime IS NULL THEN NULL
WHEN estimated_order_delivery_datetime_website_sla > orders.operation_uploaded_bank_slip_datetime THEN 1
ELSE 0
END AS is_on_time_delivery_website_sla,
-- On time delivery boolean (Customer Expectation)
CASE
WHEN order_completed_datetime IS NULL THEN NULL
WHEN estimated_order_delivery_datetime_customer_expectation > order_completed_datetime THEN 1
-- On-time boolean is null when order is not complete
WHEN account_verified_payment_datetime IS NULL THEN NULL
WHEN estimated_order_delivery_datetime_customer_expectation > orders.operation_uploaded_bank_slip_datetime THEN 1
ELSE 0
END AS is_on_time_delivery_customer_expectation,
orders.order_approved_datetime,
orders.order_completed_datetime,
orders.order_rejected_datetime,
orders.order_suspended_datetime,
orders.next_order_created_datetime,
orders.order_expected_expired_datetime,
orders.order_expired_datetime,
@@ -171,6 +187,8 @@ fct_and_dim_joins AS (
orders.cost_tax_rm,
orders.cost_service_charge_rm,
orders.total_cost_value_rm,
orders.purchase_order_number,
orders.invoice_number,
orders.cost_created_datetime,
orders.cost_pending_submission_datetime,
orders.cost_pending_verification_datetime,
@@ -178,6 +196,19 @@ fct_and_dim_joins AS (
orders.cost_completed_datetime,
orders.cost_rejected_datetime,
orders.customer_made_booking_datetime,
orders.customer_placed_order_datetime,
orders.customer_made_payment_datetime,
orders.account_verified_payment_datetime,
orders.first_white_form_generated_datetime,
orders.last_white_form_generated_datetime,
orders.operation_uploaded_bank_slip_datetime,
orders.first_purchase_order_uploaded_datetime,
orders.last_purchase_order_uploaded_datetime,
orders.operation_generated_invoice_datetime,
orders.account_rejected_payment_datetime,
orders.next_order_created_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM orders
@@ -220,6 +251,8 @@ final__rep_exchange__daily_orders AS (
-- dimensions
company_name,
supplier_company_name,
transaction_status,
document_status,
fix_currency_name,
quote_currency_name,
base_currency_name,
@@ -243,6 +276,8 @@ final__rep_exchange__daily_orders AS (
company_latitude,
company_longitude,
company_has_wallet,
company_lifetime_value,
company_m_score_lifetime,
cost_transaction_type,
cost_payment_method,
cost_status,
@@ -255,6 +290,8 @@ final__rep_exchange__daily_orders AS (
is_first_time_booking_user_completed,
is_on_time_delivery_website_sla,
is_on_time_delivery_customer_expectation,
purchase_order_number,
invoice_number,
-- measures
order_base_to_quote_currency_exchange_rate,
@@ -279,10 +316,9 @@ final__rep_exchange__daily_orders AS (
estimate_booking_quote_value,
estimate_booking_base_value,
estimate_booking_value_rm,
company_lifetime_value,
m_score_lifetime,
-- date/times
company_created_datetime,
order_created_datetime,
order_created_week,
order_expected_expired_datetime,
@@ -293,7 +329,6 @@ final__rep_exchange__daily_orders AS (
order_rejected_datetime,
order_suspended_datetime,
order_expired_datetime,
next_order_created_datetime,
cost_created_datetime,
cost_pending_submission_datetime,
cost_pending_verification_datetime,
@@ -307,6 +342,19 @@ final__rep_exchange__daily_orders AS (
estimated_order_delivery_datetime_website_sla,
estimated_order_delivery_datetime_customer_expectation,
customer_made_booking_datetime,
customer_placed_order_datetime,
customer_made_payment_datetime,
account_verified_payment_datetime,
first_white_form_generated_datetime,
last_white_form_generated_datetime,
operation_uploaded_bank_slip_datetime,
first_purchase_order_uploaded_datetime,
last_purchase_order_uploaded_datetime,
operation_generated_invoice_datetime,
account_rejected_payment_datetime,
next_order_created_datetime,
-- metadata
_dbt_ran_datetime
models/marts/reporting/rep_exchange__survival_analysis.sql
+160
View File
@@ -0,0 +1,160 @@
-- Survival analysis code reference from https://www.crosstab.io/articles/sql-survival-curves/
-- AVAILABLE FILTER VALUE
-- Source:preset_custom_filter Column:day_use_to_churn
{% set day_use_to_churn = 61 %}
-- IMPORT
WITH companies AS (
SELECT * FROM {{ ref('dim_exchange__companies') }}
),
transaction_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
-- LOGIC
-- duration_table cte computes the survival time and customer churned indicator
duration_table AS (
SELECT
companies.company_id, --subject
companies.company_created_datetime, -- event start datetime
COUNT(transaction_orders.order_id) AS count_order,
COUNT(IFF(transaction_orders.order_status = 'COMPLETED', transaction_orders.order_id, null)) AS count_completed_order,
MAX(transaction_orders.order_created_datetime) AS last_order_datetime,
COALESCE(last_order_datetime, company_created_datetime) AS last_activity_datetime,
IFF(last_order_datetime IS NULL, 1, 0) AS is_never_order_company,
-- if is_churn_company = 0, the data will be censored
IFF(DATEDIFF(day, last_activity_datetime, CURRENT_DATE()) >= {{day_use_to_churn}}, 1, 0 ) AS is_churn_company,
CASE
WHEN is_churn_company = 1 THEN
DATEDIFF(day, companies.company_created_datetime, last_activity_datetime ) + {{day_use_to_churn}}
ELSE
DATEDIFF(day, companies.company_created_datetime, CURRENT_DATE())
END AS survival_time_days -- event duration
FROM companies
LEFT JOIN transaction_orders
ON (companies.company_id = transaction_orders.company_id)
GROUP BY
companies.company_id,
companies.company_created_datetime
HAVING
-- companies without order will not be relevant to the analysis
is_never_order_company = 0
),
-- the daily_tally cte count the total number of observations at each survival_time_day
-- and the number of company that have churned at that survival_time_day
daily_observation_tally AS (
SELECT
survival_time_days,
COUNT(survival_time_days) AS total_number_of_observations,
SUM(is_churn_company) AS events -- considering only churned company
FROM duration_table
GROUP BY survival_time_days
ORDER BY survival_time_days
),
-- the cumulative_tally cte counts the number of subjects still at risk of experiencing churn
cumulative_tally AS (
SELECT
survival_time_days,
events,
total_number_of_observations,
( SELECT COUNT( DISTINCT(company_id) ) FROM duration_table ) AS total_number_of_subjects,
-- cumulative sum of observations at all previous survival_time_days SUBTRACTED by total_number_of_subjects
total_number_of_subjects - COALESCE(SUM(total_number_of_observations) OVER (
ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0
) AS at_risk
FROM daily_observation_tally
),
-- At each survival_time_day, count number of censored subject
-- censored subjects = # subject at risk - churned - # subject at risk in the next duration
compute_censored_subjects AS (
SELECT
total_number_of_subjects,
survival_time_days,
at_risk,
total_number_of_observations,
events,
at_risk - events - COALESCE(LEAD(at_risk, 1) OVER (ORDER BY survival_time_days), 0) AS censored
FROM cumulative_tally
-- Simply subtracting events from number of observations would incorrectly ignore subjects censored at durations that are dropped from the output table
WHERE events > 0
),
compute_probability AS (
SELECT
*,
-- The survival probability represents the probability of customers that will not churn up to a specific tenure
-- Example: survival_day = 61, survival_prob = 95%. For customers with 61 days of tenure, the customer has a 95% chance of not churning.
EXP(SUM(LN(1 - events / at_risk)) OVER (
ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
)) AS survival_probability,
100 * (1 - EXP(SUM(LN(1 - events / at_risk)) OVER (
ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
))) AS conversion_percentage,
SUM(events / at_risk) OVER (
ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
) AS cumulative_hazard,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM compute_censored_subjects
),
-- FINAL
final__rep_exchange__survival_analysis AS (
SELECT
-- dimension
survival_time_days,
at_risk,
total_number_of_observations,
events,
censored,
-- measures
survival_probability,
conversion_percentage,
cumulative_hazard,
-- metadata
_dbt_ran_datetime
FROM compute_probability
)
SELECT * FROM final__rep_exchange__survival_analysis
models/marts/warehouse/dim__dates.py
+2 -2
View File
@@ -62,7 +62,7 @@ def model(dbt, session):
df_work["ADD_1_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 1, df_holiday_list, 0), axis=1)
df_work["ADD_2_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 2, df_holiday_list, 0), axis=1)
df_work["ADD_3_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 3, df_holiday_list, 0), axis=1)
df_work["ADD_5_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 5, df_holiday_list, 0), axis=1)
df_work["ADD_7_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 7, df_holiday_list, 0), axis=1)
df_work["ADD_30_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 30, df_holiday_list, 0), axis=1)
df_work["ADD_90_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 90, df_holiday_list, 0), axis=1)
df_work["ADD_365_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 365, df_holiday_list, 0), axis=1)
@@ -72,7 +72,7 @@ def model(dbt, session):
df_work["ADD_1_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 1, df_holiday_list, 1), axis=1)
df_work["ADD_2_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 2, df_holiday_list, 1), axis=1)
df_work["ADD_3_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 3, df_holiday_list, 1), axis=1)
df_work["ADD_5_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 5, df_holiday_list, 1), axis=1)
df_work["ADD_7_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 7, df_holiday_list, 1), axis=1)
df_work["ADD_30_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 30, df_holiday_list, 1), axis=1)
df_work["ADD_90_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 90, df_holiday_list, 1), axis=1)
df_work["ADD_365_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 365, df_holiday_list, 1), axis=1)
models/marts/warehouse/fct_exchange__transaction_orders.sql
+181 -60
View File
@@ -9,6 +9,7 @@
column='status')
%}
-- IMPORTS
WITH transaction_orders AS (
SELECT * FROM {{ ref('int_exchange__transaction_orders_get_user_ids') }}
@@ -19,11 +20,7 @@ transaction_order_logs AS (
),
transaction_costs AS (
SELECT * FROM {{ ref('stg_exchange__transaction_costs') }}
),
transaction_cost_logs AS (
SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }}
SELECT * FROM {{ ref('int_exchange__transaction_cost_with_latest_cost_ids') }}
),
bookings AS (
@@ -34,13 +31,24 @@ booking_logs AS (
SELECT * FROM {{ ref('stg_exchange__booking_logs') }}
),
purchase_orders AS (
SELECT * FROM {{ ref('stg_exchange__transaction_purchase_orders') }}
),
invoices AS (
SELECT * FROM {{ ref('stg_exchange__transaction_invoices') }}
),
-- LOGIC
order_logs_join_bookings AS (
SELECT
transaction_order_logs.*,
bookings.service_type,
ROW_NUMBER() OVER (PARTITION BY transaction_order_id
ORDER BY transaction_order_logs.updated_datetime DESC,
ORDER BY
transaction_order_logs.updated_datetime DESC,
transaction_order_logs.transaction_order_id DESC,
transaction_order_logs.transaction_order_log_id DESC) AS rank_index
@@ -177,22 +185,7 @@ transaction_order_datetime_imputation AS (
END AS new_order_completed_datetime
FROM transaction_order_order_status_datetime
),
transaction_cost_status_datetime AS (
SELECT
transaction_cost_id,
MIN(created_datetime) AS cost_created_datetime,
{%- for status in order_status %}
MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%}
{% endfor %}
FROM transaction_cost_logs
GROUP BY
transaction_cost_id
),
booking_status_datetime AS (
@@ -209,6 +202,7 @@ booking_status_datetime AS (
GROUP BY
booking_id
),
transaction_orders_lists AS (
@@ -284,63 +278,65 @@ transaction_orders_lists AS (
LEFT JOIN transaction_order_datetime_imputation
ON (transaction_orders.transaction_order_id = transaction_order_datetime_imputation.transaction_order_id)
),
transaction_costs_lists AS (
SELECT
transaction_costs.transaction_cost_id,
transaction_costs.transaction_order_id,
transaction_costs.supplier_company_id,
transaction_costs.bank_id,
transaction_costs.base_currency_id,
transaction_costs.quote_currency_id,
transaction_costs.transaction_type,
transaction_costs.payment_method,
transaction_costs.status,
transaction_costs.payment_reference,
transaction_costs.bill_number,
transaction_costs.base_value,
transaction_costs.quote_value,
transaction_costs.base_to_quote_currency_exchange_rate,
transaction_costs.base_tax,
transaction_costs.base_service_charge,
transaction_cost_id,
transaction_order_id,
supplier_company_id,
bank_id,
base_currency_id,
quote_currency_id,
transaction_type,
payment_method,
status,
payment_reference,
bill_number,
base_value,
quote_value,
base_to_quote_currency_exchange_rate,
base_tax,
base_service_charge,
-- use for future if base currency id other from '1'
IFF(transaction_costs.base_currency_id = '1',
transaction_costs.base_value,
IFF(base_currency_id = '1',
base_value,
-9999999999) AS value_rm,
-- use for future if base currency id other from '1'
IFF(transaction_costs.base_currency_id = '1',
transaction_costs.base_tax,
IFF(base_currency_id = '1',
base_tax,
-9999999999) AS tax_rm,
-- use for future if base currency id other from '1'
IFF(transaction_costs.base_currency_id = '1',
transaction_costs.base_service_charge,
IFF(base_currency_id = '1',
base_service_charge,
-9999999999) AS service_charge_rm,
(value_rm + service_charge_rm + tax_rm) AS total_value_rm,
transaction_costs.expired_datetime,
transaction_costs.deleted_datetime,
transaction_costs.created_datetime,
transaction_costs.updated_datetime,
{%- for status in order_status %}
transaction_cost_status_datetime.cost_{{status.lower()}}_datetime,
{% endfor %}
transaction_cost_status_datetime.cost_created_datetime
expired_datetime,
deleted_datetime,
created_datetime,
updated_datetime,
cost_created_datetime,
cost_completed_datetime,
cost_pending_submission_datetime,
cost_approved_datetime,
cost_pending_verification_datetime,
cost_rejected_datetime,
cost_suspended_datetime,
cost_expired_datetime
FROM transaction_costs
LEFT JOIN transaction_cost_status_datetime
ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id)
),
bookings_lists AS (
SELECT
bookings.*,
@@ -370,6 +366,7 @@ bookings_lists AS (
LEFT JOIN booking_status_datetime
ON (bookings.booking_id = booking_status_datetime.booking_id)
),
bookings_orders_costs_join AS (
@@ -452,9 +449,7 @@ bookings_orders_costs_join AS (
transaction_costs_lists.cost_pending_verification_datetime,
transaction_costs_lists.cost_approved_datetime,
transaction_costs_lists.cost_completed_datetime,
transaction_costs_lists.cost_rejected_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
transaction_costs_lists.cost_rejected_datetime
FROM transaction_orders_lists
@@ -466,6 +461,112 @@ bookings_orders_costs_join AS (
),
purchase_orders_invoices_join AS (
SELECT
bookings_orders_costs_join.*,
invoices.created_datetime AS invoice_created_datetime,
purchase_orders.created_datetime AS purchase_order_created_datetime,
purchase_orders.updated_datetime AS purchase_order_updated_datetime,
CASE
WHEN order_status NOT IN ('REJECTED', 'SUSPENDED', 'EXPIRED') THEN purchase_orders.purchase_order_number
END AS purchase_order_number,
CASE
WHEN order_status NOT IN ('REJECTED', 'SUSPENDED', 'EXPIRED') THEN purchase_orders.status
END AS purchase_order_status,
CASE
WHEN cost_status IN ('APPROVED') THEN invoices.invoice_number
END AS invoice_number,
CASE
WHEN cost_status IN ('APPROVED') THEN invoices.status
END AS invoice_status
FROM bookings_orders_costs_join
LEFT JOIN purchase_orders
ON (bookings_orders_costs_join.booking_id = purchase_orders.booking_id)
LEFT JOIN invoices
ON (bookings_orders_costs_join.booking_id = invoices.booking_id)
),
purchase_order_invoice_status_redefined AS (
SELECT
*,
COALESCE( 'COST_' ||cost_status, 'ORDER_' || order_status ) AS transaction_temp_status,
-- Transaction status tracks the orders and costs
CASE
WHEN order_status IN ('COMPLETED', 'APPROVED') AND cost_status IS NULL
THEN 'PENDING_GENERATE_WHITE_FORM'
WHEN order_status IN ('COMPLETED', 'APPROVED') AND cost_status IN ('PENDING_VERIFICATION')
THEN 'GENERATED_WHITE_FORM'
WHEN transaction_temp_status IN ('COST_APPROVED', 'COST_COMPLETED')
THEN 'UPLOADED_BANK_SLIP'
WHEN transaction_temp_status IN ('ORDER_APPROVED')
THEN 'VERIFIED_PAYMENT_STATEMENT'
WHEN transaction_temp_status IN ('ORDER_PENDING_VERIFICATION')
THEN 'PENDING_VERIFICATION_PAYMENT_STATEMENT'
WHEN transaction_temp_status IN ('ORDER_PENDING_SUBMISSION')
THEN 'PENDING_SUBMISSION_PAYMENT_STATEMENT'
END AS transaction_temp_status_2,
-- Document status tracks the purchase orders and invoices
CASE
WHEN order_status IN ('REJECTED', 'SUSPENDED', 'EXPIRED')
THEN null
WHEN ( purchase_order_status IS NULL OR purchase_order_status IN ('PENDING_SUBMISSION') ) AND purchase_order_number IS NULL
THEN 'PENDING_SUBMISSION_PURCHASE_ORDER'
WHEN purchase_order_status IN ('PENDING_VERIFICATION')
THEN 'PENDING_VERIFICATION_PURCHASE_ORDER'
WHEN purchase_order_number IS NOT NULL AND purchase_order_status IN ( 'PENDING_VERIFICATION', 'PENDING_SUBMISSION' ) AND purchase_order_number IS NOT NULL
THEN 'PENDING_RESUBMISSION_PURCHASE_ORDER'
WHEN purchase_order_status IN ('APPROVED') AND invoice_number IS NULL AND cost_status IN ('APPROVED', 'COMPLETED')
THEN 'PENDING_GENERATE_INVOICE'
WHEN purchase_order_status IN ('APPROVED') AND invoice_status IS NULL
THEN 'APPROVED_PURCHASE_ORDER'
WHEN invoice_status IN ('APPROVED') AND cost_status IN ('COMPLETED', 'APPROVED')
THEN 'GENERATED_INVOICE'
END AS document_status,
COALESCE(transaction_temp_status_2, transaction_temp_status ) AS transaction_status
FROM purchase_orders_invoices_join
),
renaming_datetime_columns AS (
SELECT
*,
booking_created_datetime AS customer_made_booking_datetime,
order_created_datetime AS customer_placed_order_datetime,
order_pending_verification_datetime AS customer_made_payment_datetime,
order_approved_datetime AS account_verified_payment_datetime,
order_completed_datetime AS first_white_form_generated_datetime,
cost_pending_verification_datetime AS last_white_form_generated_datetime,
COALESCE(cost_approved_datetime, cost_completed_datetime) AS operation_uploaded_bank_slip_datetime,
purchase_order_created_datetime AS first_purchase_order_uploaded_datetime,
purchase_order_updated_datetime AS last_purchase_order_uploaded_datetime,
invoice_created_datetime AS operation_generated_invoice_datetime,
order_rejected_datetime AS account_rejected_payment_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM purchase_order_invoice_status_redefined
),
-- FINAL
final_fct_exchange__new_orders AS (
@@ -485,6 +586,8 @@ final_fct_exchange__new_orders AS (
base_currency_id,
-- dimensions
transaction_status,
document_status,
service_type,
booking_status,
is_first_time_booking_company,
@@ -505,6 +608,10 @@ final_fct_exchange__new_orders AS (
cost_status,
cost_payment_reference,
cost_bill_number,
purchase_order_status,
purchase_order_number,
invoice_status,
invoice_number,
-- measures
estimate_booking_base_to_quote_currency_exchange_rate,
@@ -520,6 +627,7 @@ final_fct_exchange__new_orders AS (
order_service_charge_rm,
order_tax_rm,
total_order_value_rm,
cost_base_value,
cost_quote_value,
cost_base_to_quote_currency_exchange_rate,
@@ -548,7 +656,6 @@ final_fct_exchange__new_orders AS (
order_suspended_datetime,
order_expired_datetime,
next_order_created_datetime,
cost_created_datetime,
cost_pending_submission_datetime,
cost_pending_verification_datetime,
@@ -556,10 +663,24 @@ final_fct_exchange__new_orders AS (
cost_completed_datetime,
cost_rejected_datetime,
customer_made_booking_datetime,
customer_placed_order_datetime,
customer_made_payment_datetime,
account_verified_payment_datetime,
first_white_form_generated_datetime,
last_white_form_generated_datetime,
operation_uploaded_bank_slip_datetime,
first_purchase_order_uploaded_datetime,
last_purchase_order_uploaded_datetime,
operation_generated_invoice_datetime,
account_rejected_payment_datetime,
next_order_created_datetime,
-- metadata
_dbt_ran_datetime
FROM bookings_orders_costs_join
FROM renaming_datetime_columns
)
SELECT * FROM final_fct_exchange__new_orders
models/staging/exchange/stg_exchange__companies.sql
+36 -3
View File
@@ -34,8 +34,7 @@ companies_map_constant AS (
companies.deleted_datetime,
companies.created_datetime,
companies.updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
companies.updated_datetime
FROM companies
@@ -49,6 +48,40 @@ companies_map_constant AS (
ON (companies.status = status.id)
),
/*
Data cleaning for the 4 company_id (1405, 2030, 3095, 4275) which has a company registration date later than the first order_created_datetime.
Median number of days to placing first order (4 days) is used to revert the company created datetime.
*/
data_clean AS (
SELECT
company_id,
company_marking_id,
autocount_id,
name,
company_type,
business_type,
status,
deleted_datetime,
updated_datetime,
CASE
-- 2030 has an order placed 5 months before the registration date, so using the first order_datetime as base
WHEN company_id = 2030 THEN DATEADD(day, -4, TO_TIMESTAMP_TZ('2021-06-28T14:04:30+08:00'))
WHEN company_id = 1405 THEN DATEADD(day, -4, created_datetime)
WHEN company_id = 3095 THEN DATEADD(day, -4, created_datetime)
WHEN company_id = 4275 THEN DATEADD(day, -4, created_datetime)
ELSE created_datetime
END AS created_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM companies_map_constant
),
-- FINAL
final__stg_companies__exchange AS (
@@ -74,7 +107,7 @@ final__stg_companies__exchange AS (
-- metadata
_dbt_ran_datetime
FROM companies_map_constant
FROM data_clean
)
models/staging/exchange/stg_exchange__transaction_cost_logs.sql
+3 -3
View File
@@ -20,23 +20,22 @@ status AS (
WHERE category = 'DEFAULT'
),
-- LOGIC
transactions_generate_transaction_log_id AS (
SELECT
MD5_NUMBER_LOWER64(CONCAT(transactions.transaction_id, transactions.created_datetime)) AS transaction_log_id,
MD5_NUMBER_LOWER64(CONCAT(transaction_id, created_datetime)) AS transaction_log_id,
*
FROM transactions
),
transaction_arch_logs_union_transactions AS (
SELECT * FROM transactions_generate_transaction_log_id
UNION
SELECT * FROM transaction_arch_logs
),
union_transaction_rename_column_and_column_value AS (
SELECT
@@ -96,6 +95,7 @@ union_transaction_rename_column_and_column_value AS (
transaction_arch_logs_union_transactions.transaction_type = '3' --BILL
),
-- FINAL
final__stg_exchange__transaction_log_costs AS (
models/staging/exchange/stg_exchange__transaction_costs.sql
+2 -23
View File
@@ -73,27 +73,6 @@ transactions_join_transaction_types_payment_methods_currencies_status AS (
),
remove_deleted_row AS (
SELECT
*
FROM transactions_join_transaction_types_payment_methods_currencies_status
WHERE deleted_datetime IS NULL
),
-- It will seldom offur system error, so we assume that the latest cost is correct. But it is not a correct assumption
remove_system_error_duplicate_row AS (
SELECT
*,
ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index
FROM remove_deleted_row
QUALIFY
row_number_index = 1
),
-- FINAL
final__stg_exchange__transaction_costs AS (
@@ -130,7 +109,7 @@ final__stg_exchange__transaction_costs AS (
-- metadata
_dbt_ran_datetime
FROM remove_system_error_duplicate_row
FROM transactions_join_transaction_types_payment_methods_currencies_status
)
SELECT * FROM remove_system_error_duplicate_row
SELECT * FROM final__stg_exchange__transaction_costs
models/staging/exchange/stg_exchange__transaction_invoices.sql
+145
View File
@@ -0,0 +1,145 @@
/* DOCS
Invoice is issued by CIEF to customers for customer book keeping.
An invoice is a document that consolidates the total amount for multiple orders within a single booking.
Currency shown in invoice is the converted currency.
For booking_id with multiple invoices, take latest updated_datetime as true.
The duplicates are a bug as confirmed by omair.
*/
-- IMPORTS
WITH transactions AS (
SELECT * FROM {{ ref('base_exchange__transactions') }}
),
transaction_types AS (
SELECT * FROM {{ ref('seed_exchange__transaction_types') }}
),
bookings AS (
SELECT * FROM {{ ref('base_exchange__bookings') }}
),
payment_methods AS (
SELECT * FROM {{ ref('seed_exchange__payment_methods') }}
),
status AS (
SELECT * FROM {{ ref('seed_exchange__status') }}
WHERE category = 'DEFAULT'
),
-- LOGIC
transactions_join_transaction_types_payment_methods_bookings_status AS (
SELECT
bookings.booking_marking_id,
bookings.fix_value,
bookings.fix_currency_id,
transactions.transaction_id AS transaction_invoice_id,
transactions.owner_id AS booking_id,
transactions.issuer_user_id AS issuer_company_id,
transactions.receiver_user_id AS receiver_company_id,
transactions.base_currency_id,
transactions.quote_currency_id,
COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type,
COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method,
COALESCE(status.name, transactions.status::string) AS status,
transactions.bill_number AS invoice_number,
transactions.base_value,
transactions.quote_value,
transactions.base_to_quote_currency_exchange_rate,
transactions.base_tax,
transactions.base_service_charge,
transactions.expired_datetime,
transactions.deleted_datetime,
transactions.created_datetime,
transactions.updated_datetime
FROM transactions
LEFT JOIN transaction_types
ON (transactions.transaction_type = transaction_types.id)
LEFT JOIN payment_methods
ON (transactions.payment_method = payment_methods.id)
LEFT JOIN status
ON (transactions.status = status.id)
LEFT JOIN bookings
ON (transactions.owner_id = bookings.booking_id)
WHERE
transactions.owner_type = 'App\\Models\\Booking'
AND
transactions.transaction_type = '2' --INVOICE
),
remove_deleted_records AS (
SELECT
*
FROM transactions_join_transaction_types_payment_methods_bookings_status
WHERE deleted_datetime IS NULL
),
remove_duplicated_records AS (
SELECT
*,
ROW_NUMBER() OVER (PARTITION BY booking_id ORDER BY updated_datetime DESC) AS row_index,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM remove_deleted_records
QUALIFY
row_index = 1
),
-- FINAL
final__stg_exchange__invoices AS (
SELECT
-- ids
transaction_invoice_id,
booking_id,
booking_marking_id,
issuer_company_id,
receiver_company_id,
base_currency_id,
quote_currency_id,
fix_currency_id,
-- dimensions
invoice_number,
transaction_type,
payment_method,
status,
-- measures
fix_value,
base_value,
quote_value,
base_to_quote_currency_exchange_rate,
base_tax,
base_service_charge,
-- date/times
deleted_datetime,
created_datetime,
expired_datetime,
updated_datetime,
-- metadata
_dbt_ran_datetime
FROM remove_duplicated_records
)
SELECT * FROM final__stg_exchange__invoices
models/staging/exchange/stg_exchange__transaction_purchase_orders.sql
+149
View File
@@ -0,0 +1,149 @@
/* DOCS
"IS_AUTO_GENERATED_PURCHASE_ORDER"
- 1 indicates PO auto generated by system, having prefix 'XPO'
- 0 indicates PO submitted by customer, having prefix 'PO'
Purchase Order document does not perform currency conversion, and it is a document to be submitted by the customer.
Therefore, base currency and quote currency is the same.
The PO is based on the fix currency.
The 'fix currency' feature enables customers to choose their primary currency when placing an order,
ensuring that it remains fixed while allowing the converted currency to fluctuate.
Currency is converted only in invoice.
Note to downstream user:
This table contains duplicate of booking_id, due to deleted entries.
Filter out deleted entries and select only status = 'APPROVED' for use.
*/
-- IMPORTS
WITH transactions AS (
SELECT * FROM {{ ref('base_exchange__transactions') }}
),
transaction_types AS (
SELECT * FROM {{ ref('seed_exchange__transaction_types') }}
),
bookings AS (
SELECT * FROM {{ ref('base_exchange__bookings') }}
),
payment_methods AS (
SELECT * FROM {{ ref('seed_exchange__payment_methods') }}
),
status AS (
SELECT * FROM {{ ref('seed_exchange__status') }}
WHERE category = 'DEFAULT'
),
-- LOGIC
transactions_join_transaction_types_payment_methods_bookings_status AS (
SELECT
bookings.booking_marking_id,
bookings.fix_value,
bookings.fix_currency_id AS purchase_order_currency_id,
transactions.transaction_id AS transaction_purchase_order_id,
transactions.owner_id AS booking_id,
transactions.issuer_user_id AS issuer_company_id,
transactions.receiver_user_id AS receiver_company_id,
COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type,
COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method,
COALESCE(status.name, transactions.status::string) AS status,
transactions.bill_number AS purchase_order_number,
transactions.base_value AS purchase_order_amount,
transactions.expired_datetime,
transactions.deleted_datetime,
transactions.created_datetime,
transactions.updated_datetime,
CASE
WHEN purchase_order_number LIKE 'XP%' THEN 1 ELSE 0
END AS is_auto_generated_purchase_order
FROM transactions
LEFT JOIN transaction_types
ON (transactions.transaction_type = transaction_types.id)
LEFT JOIN payment_methods
ON (transactions.payment_method = payment_methods.id)
LEFT JOIN status
ON (transactions.status = status.id)
LEFT JOIN bookings
ON (transactions.owner_id = bookings.booking_id)
WHERE
transactions.owner_type = 'App\\Models\\Booking'
AND
transactions.transaction_type = '7' --PURCHASE_ORDER
),
remove_deleted_records AS (
SELECT
*
FROM transactions_join_transaction_types_payment_methods_bookings_status
WHERE deleted_datetime IS NULL
),
remove_duplicated_records AS (
SELECT
*,
ROW_NUMBER() OVER (PARTITION BY booking_id ORDER BY updated_datetime DESC) AS row_index,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM remove_deleted_records
QUALIFY
row_index = 1
),
-- FINAL
final__stg_exchange__purchase_orders AS (
SELECT
-- ids
transaction_purchase_order_id,
booking_id,
booking_marking_id,
issuer_company_id,
receiver_company_id,
purchase_order_currency_id
-- dimensions
is_auto_generated_purchase_order,
purchase_order_number,
transaction_type,
payment_method,
status,
-- measures
fix_value,
purchase_order_amount,
-- date/times
deleted_datetime,
created_datetime,
expired_datetime,
updated_datetime,
-- metadata
_dbt_ran_datetime
FROM remove_duplicated_records
)
SELECT * FROM final__stg_exchange__purchase_orders