From 631e7769b95533635b960fbe1a1c5c99566ef03c Mon Sep 17 00:00:00 2001 From: CIEF DataTeam Date: Mon, 31 Jul 2023 16:08:35 +0000 Subject: [PATCH 1/7] Migrate Metrics from REP to FCT --- ...rigin_warehouse_shipping_packing_lists.sql | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql b/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql index a0c3738..f597e4f 100644 --- a/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql +++ b/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql @@ -159,9 +159,7 @@ origin_warehouse_shipping_packing_lists_enrich AS ( destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_last_eta_datetime, destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_first_estimate_schedule_created_datetime, destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_last_estimate_schedule_created_datetime, - destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_created_datetime, - - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_created_datetime FROM origin_warehouse_shipping_packing_lists @@ -185,6 +183,33 @@ origin_warehouse_shipping_packing_lists_enrich AS ( ON (origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = destination_warehouse_packing_list_transport_arrangements.old_origin_warehouse_shipping_packing_list_id) ), +create_new_column AS ( + SELECT + *, + (ZEROIFNULL(total_package_cbm) + + ZEROIFNULL(over_weight_cbm) + + ZEROIFNULL(pallet_charge_cbm)) AS total_order_cbm, + + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY sub_company_id ORDER BY origin_warehouse_shipping_order_created_datetime) = 1 THEN 1 + ELSE 0 + END AS is_first_time_sub_company, + + CASE + WHEN origin_warehouse_shipping_order_status = 'COMPLETED' + AND + MIN(IFF(origin_warehouse_shipping_order_status = 'COMPLETED', origin_warehouse_shipping_order_created_datetime, NULL)) + OVER (PARTITION BY sub_company_id) + = origin_warehouse_shipping_order_created_datetime + THEN 1 + ELSE 0 + END AS is_first_time_sub_company_completed, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM origin_warehouse_shipping_packing_lists_enrich +), + -- FINAL final__fct_shipping__origin_warehouse_shipping_packing_lists AS ( @@ -206,6 +231,8 @@ final__fct_shipping__origin_warehouse_shipping_packing_lists AS ( warehouse_order_number, reference_contract, has_replaced_by_new_origin_warehouse_shipping_packing_list_id, + is_first_time_sub_company, + is_first_time_sub_company_completed, service_type, package_descriptions, delivery_address_reference, @@ -256,6 +283,7 @@ final__fct_shipping__origin_warehouse_shipping_packing_lists AS ( over_weight_kg, number_of_pallet_charge_record, pallet_charge_cbm, + total_order_cbm, number_of_invoice, number_of_payment, invoice_number_of_shipping_fee, @@ -321,7 +349,7 @@ final__fct_shipping__origin_warehouse_shipping_packing_lists AS ( -- metadata _dbt_ran_datetime - FROM origin_warehouse_shipping_packing_lists_enrich + FROM create_new_column ) SELECT * FROM final__fct_shipping__origin_warehouse_shipping_packing_lists \ No newline at end of file From 66b79b6ececc8c905be24c2bd543aa52cf9ffdf3 Mon Sep 17 00:00:00 2001 From: CIEF DataTeam Date: Mon, 31 Jul 2023 16:08:46 +0000 Subject: [PATCH 2/7] Migrate Metrics from REP to FCT --- ..._shipping__origin_warehouse_shipping_orders.sql | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql b/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql index 5285d51..f7a83e9 100644 --- a/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql +++ b/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql @@ -26,25 +26,21 @@ join_tables AS ( create_metrics AS ( SELECT *, - {% for column_name in ["parent_company", "sub_company"] %} + CASE - WHEN ROW_NUMBER() OVER (PARTITION BY join_tables.{{column_name}}_id ORDER BY join_tables.origin_warehouse_shipping_order_created_datetime) = 1 THEN 1 + WHEN ROW_NUMBER() OVER (PARTITION BY join_tables.parent_company_id ORDER BY join_tables.origin_warehouse_shipping_order_created_datetime) = 1 THEN 1 ELSE 0 - END AS is_first_time_{{column_name}}, + END AS is_first_time_parent_company, CASE WHEN join_tables.origin_warehouse_shipping_order_status = 'COMPLETED' AND MIN(IFF(join_tables.origin_warehouse_shipping_order_status = 'COMPLETED', join_tables.origin_warehouse_shipping_order_created_datetime, NULL)) - OVER (PARTITION BY join_tables.{{column_name}}_id) + OVER (PARTITION BY join_tables.parent_company_id) = join_tables.origin_warehouse_shipping_order_created_datetime THEN 1 ELSE 0 - END AS is_first_time_{{column_name}}_completed, - {% endfor %} - - -- temp debug, need make sure it do not have null in fct, cause cant add - (ZEROIFNULL(total_package_cbm) + ZEROIFNULL(over_weight_cbm) + ZEROIFNULL(pallet_charge_cbm)) AS total_order_cbm, + END AS is_first_time_parent_company_completed, '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime From 86115d7fa0ec32fa2bfc624dc3c8a3a98e90a190 Mon Sep 17 00:00:00 2001 From: CIEF DataTeam Date: Mon, 31 Jul 2023 16:09:04 +0000 Subject: [PATCH 3/7] Develop dashboard --- .../rep_shipping__company_order_usages.sql | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 models/marts/reporting/rep_shipping__company_order_usages.sql diff --git a/models/marts/reporting/rep_shipping__company_order_usages.sql b/models/marts/reporting/rep_shipping__company_order_usages.sql new file mode 100644 index 0000000..edc47af --- /dev/null +++ b/models/marts/reporting/rep_shipping__company_order_usages.sql @@ -0,0 +1,163 @@ +{% set time_grain_list = ["month", "quarter", "year"] %} +{% set monthly_cbm_rank_list = [1.25, 5, 20] %} -- 0->1.25 40% 1.25->5 40% 5->20 15% 20->onward 5% +{% set quarterly_cbm_rank_list = [1.75, 10, 30] %} +{% set yearly_cbm_rank_list = [2.5, 20, 60] %} + +-- IMPORT +WITH shipping_companies AS ( + SELECT * FROM {{ ref('dim_shipping__companies') }} +), + +origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('fct_shipping__origin_warehouse_shipping_packing_lists') }} +), + +dates AS ( + SELECT * FROM {{ ref('dim__dates') }} +), + +-- LOGIC +{% for time_grain in time_grain_list %} + sub_company_register_{{time_grain}} AS ( + SELECT + sub_company_id, + sub_company_marking_id, + dates.first_day_of_{{time_grain}} AS company_register_{{time_grain}} + + FROM shipping_companies + + LEFT JOIN dates + ON (date(shipping_companies.sub_company_created_datetime) = dates.date_day) + ), + + origin_warehouse_shipping_packing_list_{{time_grain}} AS ( + SELECT + origin_warehouse_shipping_packing_lists.*, + dates.first_day_of_{{time_grain}} AS origin_warehouse_shipping_order_created_{{time_grain}} + + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN dates + ON (date(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) = dates.date_day) + ), + + register_date_and_company_id_combination_{{time_grain}} AS ( + SELECT + DISTINCT c.company_register_{{time_grain}} AS event_date, + '{{time_grain|upper}}LY' AS time_grain_filter, + c1.sub_company_id, + c1.sub_company_marking_id + + FROM sub_company_register_{{time_grain}} AS c + + CROSS JOIN sub_company_register_{{time_grain}} AS c1 + + WHERE c.company_register_{{time_grain}} >= c1.company_register_{{time_grain}} + + ORDER BY event_date + ), + + company_sale_by_time_grain_{{time_grain}} AS ( + SELECT + register_date_and_company_id_combination_{{time_grain}}.event_date, + register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, + register_date_and_company_id_combination_{{time_grain}}.sub_company_id, + register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id, + + SUM(IFF(origin_warehouse_shipping_order_status = 'APPROVED' OR + origin_warehouse_shipping_order_status = 'COMPLETED', + total_order_cbm, + 0)) AS total_cbm_completed_approved, + + COUNT(DISTINCT(IFF(origin_warehouse_shipping_order_status = 'APPROVED' OR + origin_warehouse_shipping_order_status = 'COMPLETED', + old_origin_warehouse_shipping_packing_list_id, + NULL))) AS total_order_completed_approved, + + {% if time_grain == "month" -%} + {%- set cbm_rank_list = monthly_cbm_rank_list -%} + {%- elif time_grain == "quarter"%} + {%- set cbm_rank_list = quarterly_cbm_rank_list -%} + {%- elif time_grain == "year"%} + {%- set cbm_rank_list = yearly_cbm_rank_list -%} + {%- else -%} + {%- set cbm_rank_list = [] -%} + {%- endif -%} + + CASE + WHEN total_cbm_completed_approved = 0 + THEN 'NO_USAGE' + WHEN total_cbm_completed_approved <= {{cbm_rank_list[0]}} + THEN 'LOW_USAGE' + WHEN (total_cbm_completed_approved > {{cbm_rank_list[0]}} AND total_cbm_completed_approved <= {{cbm_rank_list[1]}}) + THEN 'MEDIUM_USAGE' + WHEN (total_cbm_completed_approved > {{cbm_rank_list[1]}} AND total_cbm_completed_approved <= {{cbm_rank_list[2]}}) + THEN 'HIGH_USAGE' + WHEN total_cbm_completed_approved > {{cbm_rank_list[2]}} + THEN 'SUPER_HIGH_USAGE' + ELSE 'ERROR' + END AS company_usage_type, + + CASE + WHEN total_order_completed_approved = 0 + THEN 'NO_ORDER' + WHEN MAX(is_first_time_sub_company) = 1 + THEN 'FIRST_TIME_ORDER' + WHEN LAG(total_order_completed_approved) OVER (PARTITION BY register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, register_date_and_company_id_combination_{{time_grain}}.sub_company_id, register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id ORDER BY event_date) != 0 + THEN 'RECURRING_ORDER' + WHEN LAG(total_order_completed_approved) OVER (PARTITION BY register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, register_date_and_company_id_combination_{{time_grain}}.sub_company_id, register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id ORDER BY event_date) = 0 + THEN 'REACTIVE_ORDER' + ELSE 'ERROR' + END AS event_type + + + FROM register_date_and_company_id_combination_{{time_grain}} + + LEFT JOIN origin_warehouse_shipping_packing_list_{{time_grain}} + ON (register_date_and_company_id_combination_{{time_grain}}.sub_company_id = origin_warehouse_shipping_packing_list_{{time_grain}}.sub_company_id + AND + register_date_and_company_id_combination_{{time_grain}}.event_date = origin_warehouse_shipping_packing_list_{{time_grain}}.origin_warehouse_shipping_order_created_{{time_grain}}) + + GROUP BY + register_date_and_company_id_combination_{{time_grain}}.event_date, + register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, + register_date_and_company_id_combination_{{time_grain}}.sub_company_id, + register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id + ), + +{% endfor %} + +union_all_time_grain AS ( + {% for time_grain in time_grain_list %} + SELECT * FROM company_sale_by_time_grain_{{time_grain}} + {%- if not loop.last %} UNION{% endif -%} + {% endfor %} +), + +--FINAL +final__rep_shipping__company_order_usages AS ( + SELECT + -- ids + {{ dbt_utils.generate_surrogate_key(['event_date', 'time_grain_filter', 'sub_company_id']) }} AS company_order_usage_sk, + sub_company_id, + sub_company_marking_id, + + -- dimensions + time_grain_filter, + company_usage_type, + event_type, + + -- measures + total_cbm_completed_approved, + total_order_completed_approved, + + -- date/times + event_date, + + -- metadata + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM union_all_time_grain +) + +SELECT * FROM final__rep_shipping__company_order_usages \ No newline at end of file From 5f176c61fb942f0ab0d3222ce588068f0412d9c5 Mon Sep 17 00:00:00 2001 From: CIEF DataTeam Date: Tue, 1 Aug 2023 06:56:41 +0000 Subject: [PATCH 4/7] remove unorder company --- ..._shipping__active_company_order_usages.sql | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 models/marts/reporting/rep_shipping__active_company_order_usages.sql diff --git a/models/marts/reporting/rep_shipping__active_company_order_usages.sql b/models/marts/reporting/rep_shipping__active_company_order_usages.sql new file mode 100644 index 0000000..d9e1bab --- /dev/null +++ b/models/marts/reporting/rep_shipping__active_company_order_usages.sql @@ -0,0 +1,171 @@ +{% set time_grain_list = ["month", "quarter", "year"] %} +{% set monthly_cbm_rank_list = [1.25, 5, 20] %} -- 0->1.25 40% 1.25->5 40% 5->20 15% 20->onward 5% +{% set quarterly_cbm_rank_list = [1.75, 10, 30] %} +{% set yearly_cbm_rank_list = [2.5, 20, 60] %} + +-- IMPORT +WITH shipping_companies AS ( + SELECT * FROM {{ ref('dim_shipping__companies') }} +), + +origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('fct_shipping__origin_warehouse_shipping_packing_lists') }} +), + +dates AS ( + SELECT * FROM {{ ref('dim__dates') }} +), + +-- LOGIC +{% for time_grain in time_grain_list %} + sub_company_first_time_order_{{time_grain}} AS ( + SELECT + origin_warehouse_shipping_packing_lists.sub_company_id, + shipping_companies.sub_company_marking_id, + MIN(dates.first_day_of_{{time_grain}}) AS company_first_time_order_{{time_grain}} + + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN shipping_companies + ON (origin_warehouse_shipping_packing_lists.sub_company_id = shipping_companies.sub_company_id) + + LEFT JOIN dates + ON (date(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) = dates.date_day) + + GROUP BY + origin_warehouse_shipping_packing_lists.sub_company_id, + shipping_companies.sub_company_marking_id + + ), + + origin_warehouse_shipping_packing_list_{{time_grain}} AS ( + SELECT + origin_warehouse_shipping_packing_lists.*, + dates.first_day_of_{{time_grain}} AS origin_warehouse_shipping_order_created_{{time_grain}} + + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN dates + ON (date(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) = dates.date_day) + ), + + register_date_and_company_id_combination_{{time_grain}} AS ( + SELECT + DISTINCT c.company_first_time_order_{{time_grain}} AS event_date, + '{{time_grain|upper}}LY' AS time_grain_filter, + c1.sub_company_id, + c1.sub_company_marking_id + + FROM sub_company_first_time_order_{{time_grain}} AS c + + CROSS JOIN sub_company_first_time_order_{{time_grain}} AS c1 + + WHERE c.company_first_time_order_{{time_grain}} >= c1.company_first_time_order_{{time_grain}} + + ORDER BY event_date + ), + + company_sale_by_time_grain_{{time_grain}} AS ( + SELECT + register_date_and_company_id_combination_{{time_grain}}.event_date, + register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, + register_date_and_company_id_combination_{{time_grain}}.sub_company_id, + register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id, + + SUM(IFF(origin_warehouse_shipping_order_status = 'APPROVED' OR + origin_warehouse_shipping_order_status = 'COMPLETED', + total_order_cbm, + 0)) AS total_cbm_completed_approved, + + COUNT(DISTINCT(IFF(origin_warehouse_shipping_order_status = 'APPROVED' OR + origin_warehouse_shipping_order_status = 'COMPLETED', + old_origin_warehouse_shipping_packing_list_id, + NULL))) AS total_order_completed_approved, + + {% if time_grain == "month" -%} + {%- set cbm_rank_list = monthly_cbm_rank_list -%} + {%- elif time_grain == "quarter"%} + {%- set cbm_rank_list = quarterly_cbm_rank_list -%} + {%- elif time_grain == "year"%} + {%- set cbm_rank_list = yearly_cbm_rank_list -%} + {%- else -%} + {%- set cbm_rank_list = [] -%} + {%- endif -%} + + CASE + WHEN total_cbm_completed_approved = 0 + THEN 'NO_USAGE' + WHEN total_cbm_completed_approved <= {{cbm_rank_list[0]}} + THEN 'LOW_USAGE' + WHEN (total_cbm_completed_approved > {{cbm_rank_list[0]}} AND total_cbm_completed_approved <= {{cbm_rank_list[1]}}) + THEN 'MEDIUM_USAGE' + WHEN (total_cbm_completed_approved > {{cbm_rank_list[1]}} AND total_cbm_completed_approved <= {{cbm_rank_list[2]}}) + THEN 'HIGH_USAGE' + WHEN total_cbm_completed_approved > {{cbm_rank_list[2]}} + THEN 'SUPER_HIGH_USAGE' + ELSE 'ERROR' + END AS company_usage_type, + + CASE + WHEN total_order_completed_approved = 0 + THEN 'NO_ORDER' + WHEN MAX(is_first_time_sub_company) = 1 + THEN 'FIRST_TIME_ORDER' + WHEN LAG(total_order_completed_approved) OVER (PARTITION BY register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, register_date_and_company_id_combination_{{time_grain}}.sub_company_id, register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id ORDER BY event_date) != 0 + THEN 'RECURRING_ORDER' + WHEN LAG(total_order_completed_approved) OVER (PARTITION BY register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, register_date_and_company_id_combination_{{time_grain}}.sub_company_id, register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id ORDER BY event_date) = 0 + THEN 'REACTIVE_ORDER' + ELSE 'ERROR' + END AS event_type + + + FROM register_date_and_company_id_combination_{{time_grain}} + + LEFT JOIN origin_warehouse_shipping_packing_list_{{time_grain}} + ON (register_date_and_company_id_combination_{{time_grain}}.sub_company_id = origin_warehouse_shipping_packing_list_{{time_grain}}.sub_company_id + AND + register_date_and_company_id_combination_{{time_grain}}.event_date = origin_warehouse_shipping_packing_list_{{time_grain}}.origin_warehouse_shipping_order_created_{{time_grain}}) + + GROUP BY + register_date_and_company_id_combination_{{time_grain}}.event_date, + register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, + register_date_and_company_id_combination_{{time_grain}}.sub_company_id, + register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id + ), + +{% endfor %} + +union_all_time_grain AS ( + {% for time_grain in time_grain_list %} + SELECT * FROM company_sale_by_time_grain_{{time_grain}} + {%- if not loop.last %} UNION{% endif -%} + {% endfor %} +), + +--FINAL +final__rep_shipping__company_order_usages AS ( + SELECT + -- ids + {{ dbt_utils.generate_surrogate_key(['event_date', 'time_grain_filter', 'sub_company_id']) }} AS company_order_usage_sk, + sub_company_id, + sub_company_marking_id, + + -- dimensions + time_grain_filter, + company_usage_type, + event_type, + + -- measures + total_cbm_completed_approved, + total_order_completed_approved, + + -- date/times + event_date, + + -- metadata + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM union_all_time_grain +) + +SELECT * FROM final__rep_shipping__company_order_usages \ No newline at end of file From 7e3c0cdc48868d51976ba16a1396d5bbf8d9637a Mon Sep 17 00:00:00 2001 From: CIEF DataTeam Date: Tue, 1 Aug 2023 09:08:23 +0000 Subject: [PATCH 5/7] Only with company_id who order before --- .../rep_shipping__company_order_usages.sql | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/models/marts/reporting/rep_shipping__company_order_usages.sql b/models/marts/reporting/rep_shipping__company_order_usages.sql index edc47af..d9e1bab 100644 --- a/models/marts/reporting/rep_shipping__company_order_usages.sql +++ b/models/marts/reporting/rep_shipping__company_order_usages.sql @@ -18,16 +18,24 @@ dates AS ( -- LOGIC {% for time_grain in time_grain_list %} - sub_company_register_{{time_grain}} AS ( + sub_company_first_time_order_{{time_grain}} AS ( SELECT - sub_company_id, - sub_company_marking_id, - dates.first_day_of_{{time_grain}} AS company_register_{{time_grain}} + origin_warehouse_shipping_packing_lists.sub_company_id, + shipping_companies.sub_company_marking_id, + MIN(dates.first_day_of_{{time_grain}}) AS company_first_time_order_{{time_grain}} - FROM shipping_companies + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN shipping_companies + ON (origin_warehouse_shipping_packing_lists.sub_company_id = shipping_companies.sub_company_id) LEFT JOIN dates - ON (date(shipping_companies.sub_company_created_datetime) = dates.date_day) + ON (date(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) = dates.date_day) + + GROUP BY + origin_warehouse_shipping_packing_lists.sub_company_id, + shipping_companies.sub_company_marking_id + ), origin_warehouse_shipping_packing_list_{{time_grain}} AS ( @@ -43,16 +51,16 @@ dates AS ( register_date_and_company_id_combination_{{time_grain}} AS ( SELECT - DISTINCT c.company_register_{{time_grain}} AS event_date, + DISTINCT c.company_first_time_order_{{time_grain}} AS event_date, '{{time_grain|upper}}LY' AS time_grain_filter, c1.sub_company_id, c1.sub_company_marking_id - FROM sub_company_register_{{time_grain}} AS c + FROM sub_company_first_time_order_{{time_grain}} AS c - CROSS JOIN sub_company_register_{{time_grain}} AS c1 + CROSS JOIN sub_company_first_time_order_{{time_grain}} AS c1 - WHERE c.company_register_{{time_grain}} >= c1.company_register_{{time_grain}} + WHERE c.company_first_time_order_{{time_grain}} >= c1.company_first_time_order_{{time_grain}} ORDER BY event_date ), From ba586122040d5d92f68f305cc55188aab4d39ff1 Mon Sep 17 00:00:00 2001 From: CIEF DataTeam Date: Tue, 1 Aug 2023 09:27:53 +0000 Subject: [PATCH 6/7] Duplicate Report --- ..._shipping__active_company_order_usages.sql | 171 ------------------ 1 file changed, 171 deletions(-) delete mode 100644 models/marts/reporting/rep_shipping__active_company_order_usages.sql diff --git a/models/marts/reporting/rep_shipping__active_company_order_usages.sql b/models/marts/reporting/rep_shipping__active_company_order_usages.sql deleted file mode 100644 index d9e1bab..0000000 --- a/models/marts/reporting/rep_shipping__active_company_order_usages.sql +++ /dev/null @@ -1,171 +0,0 @@ -{% set time_grain_list = ["month", "quarter", "year"] %} -{% set monthly_cbm_rank_list = [1.25, 5, 20] %} -- 0->1.25 40% 1.25->5 40% 5->20 15% 20->onward 5% -{% set quarterly_cbm_rank_list = [1.75, 10, 30] %} -{% set yearly_cbm_rank_list = [2.5, 20, 60] %} - --- IMPORT -WITH shipping_companies AS ( - SELECT * FROM {{ ref('dim_shipping__companies') }} -), - -origin_warehouse_shipping_packing_lists AS ( - SELECT * FROM {{ ref('fct_shipping__origin_warehouse_shipping_packing_lists') }} -), - -dates AS ( - SELECT * FROM {{ ref('dim__dates') }} -), - --- LOGIC -{% for time_grain in time_grain_list %} - sub_company_first_time_order_{{time_grain}} AS ( - SELECT - origin_warehouse_shipping_packing_lists.sub_company_id, - shipping_companies.sub_company_marking_id, - MIN(dates.first_day_of_{{time_grain}}) AS company_first_time_order_{{time_grain}} - - FROM origin_warehouse_shipping_packing_lists - - LEFT JOIN shipping_companies - ON (origin_warehouse_shipping_packing_lists.sub_company_id = shipping_companies.sub_company_id) - - LEFT JOIN dates - ON (date(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) = dates.date_day) - - GROUP BY - origin_warehouse_shipping_packing_lists.sub_company_id, - shipping_companies.sub_company_marking_id - - ), - - origin_warehouse_shipping_packing_list_{{time_grain}} AS ( - SELECT - origin_warehouse_shipping_packing_lists.*, - dates.first_day_of_{{time_grain}} AS origin_warehouse_shipping_order_created_{{time_grain}} - - FROM origin_warehouse_shipping_packing_lists - - LEFT JOIN dates - ON (date(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) = dates.date_day) - ), - - register_date_and_company_id_combination_{{time_grain}} AS ( - SELECT - DISTINCT c.company_first_time_order_{{time_grain}} AS event_date, - '{{time_grain|upper}}LY' AS time_grain_filter, - c1.sub_company_id, - c1.sub_company_marking_id - - FROM sub_company_first_time_order_{{time_grain}} AS c - - CROSS JOIN sub_company_first_time_order_{{time_grain}} AS c1 - - WHERE c.company_first_time_order_{{time_grain}} >= c1.company_first_time_order_{{time_grain}} - - ORDER BY event_date - ), - - company_sale_by_time_grain_{{time_grain}} AS ( - SELECT - register_date_and_company_id_combination_{{time_grain}}.event_date, - register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, - register_date_and_company_id_combination_{{time_grain}}.sub_company_id, - register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id, - - SUM(IFF(origin_warehouse_shipping_order_status = 'APPROVED' OR - origin_warehouse_shipping_order_status = 'COMPLETED', - total_order_cbm, - 0)) AS total_cbm_completed_approved, - - COUNT(DISTINCT(IFF(origin_warehouse_shipping_order_status = 'APPROVED' OR - origin_warehouse_shipping_order_status = 'COMPLETED', - old_origin_warehouse_shipping_packing_list_id, - NULL))) AS total_order_completed_approved, - - {% if time_grain == "month" -%} - {%- set cbm_rank_list = monthly_cbm_rank_list -%} - {%- elif time_grain == "quarter"%} - {%- set cbm_rank_list = quarterly_cbm_rank_list -%} - {%- elif time_grain == "year"%} - {%- set cbm_rank_list = yearly_cbm_rank_list -%} - {%- else -%} - {%- set cbm_rank_list = [] -%} - {%- endif -%} - - CASE - WHEN total_cbm_completed_approved = 0 - THEN 'NO_USAGE' - WHEN total_cbm_completed_approved <= {{cbm_rank_list[0]}} - THEN 'LOW_USAGE' - WHEN (total_cbm_completed_approved > {{cbm_rank_list[0]}} AND total_cbm_completed_approved <= {{cbm_rank_list[1]}}) - THEN 'MEDIUM_USAGE' - WHEN (total_cbm_completed_approved > {{cbm_rank_list[1]}} AND total_cbm_completed_approved <= {{cbm_rank_list[2]}}) - THEN 'HIGH_USAGE' - WHEN total_cbm_completed_approved > {{cbm_rank_list[2]}} - THEN 'SUPER_HIGH_USAGE' - ELSE 'ERROR' - END AS company_usage_type, - - CASE - WHEN total_order_completed_approved = 0 - THEN 'NO_ORDER' - WHEN MAX(is_first_time_sub_company) = 1 - THEN 'FIRST_TIME_ORDER' - WHEN LAG(total_order_completed_approved) OVER (PARTITION BY register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, register_date_and_company_id_combination_{{time_grain}}.sub_company_id, register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id ORDER BY event_date) != 0 - THEN 'RECURRING_ORDER' - WHEN LAG(total_order_completed_approved) OVER (PARTITION BY register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, register_date_and_company_id_combination_{{time_grain}}.sub_company_id, register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id ORDER BY event_date) = 0 - THEN 'REACTIVE_ORDER' - ELSE 'ERROR' - END AS event_type - - - FROM register_date_and_company_id_combination_{{time_grain}} - - LEFT JOIN origin_warehouse_shipping_packing_list_{{time_grain}} - ON (register_date_and_company_id_combination_{{time_grain}}.sub_company_id = origin_warehouse_shipping_packing_list_{{time_grain}}.sub_company_id - AND - register_date_and_company_id_combination_{{time_grain}}.event_date = origin_warehouse_shipping_packing_list_{{time_grain}}.origin_warehouse_shipping_order_created_{{time_grain}}) - - GROUP BY - register_date_and_company_id_combination_{{time_grain}}.event_date, - register_date_and_company_id_combination_{{time_grain}}.time_grain_filter, - register_date_and_company_id_combination_{{time_grain}}.sub_company_id, - register_date_and_company_id_combination_{{time_grain}}.sub_company_marking_id - ), - -{% endfor %} - -union_all_time_grain AS ( - {% for time_grain in time_grain_list %} - SELECT * FROM company_sale_by_time_grain_{{time_grain}} - {%- if not loop.last %} UNION{% endif -%} - {% endfor %} -), - ---FINAL -final__rep_shipping__company_order_usages AS ( - SELECT - -- ids - {{ dbt_utils.generate_surrogate_key(['event_date', 'time_grain_filter', 'sub_company_id']) }} AS company_order_usage_sk, - sub_company_id, - sub_company_marking_id, - - -- dimensions - time_grain_filter, - company_usage_type, - event_type, - - -- measures - total_cbm_completed_approved, - total_order_completed_approved, - - -- date/times - event_date, - - -- metadata - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - - FROM union_all_time_grain -) - -SELECT * FROM final__rep_shipping__company_order_usages \ No newline at end of file From f426376997de053c16c4652736770be7f7363cb3 Mon Sep 17 00:00:00 2001 From: CIEF DataTeam Date: Wed, 2 Aug 2023 12:17:43 +0000 Subject: [PATCH 7/7] Remove Super high usage --- .../reporting/rep_shipping__company_order_usages.sql | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/models/marts/reporting/rep_shipping__company_order_usages.sql b/models/marts/reporting/rep_shipping__company_order_usages.sql index d9e1bab..4cf1d13 100644 --- a/models/marts/reporting/rep_shipping__company_order_usages.sql +++ b/models/marts/reporting/rep_shipping__company_order_usages.sql @@ -1,7 +1,7 @@ {% set time_grain_list = ["month", "quarter", "year"] %} -{% set monthly_cbm_rank_list = [1.25, 5, 20] %} -- 0->1.25 40% 1.25->5 40% 5->20 15% 20->onward 5% -{% set quarterly_cbm_rank_list = [1.75, 10, 30] %} -{% set yearly_cbm_rank_list = [2.5, 20, 60] %} +{% set monthly_cbm_rank_list = [1.25, 5] %} -- 0->1.25 40% 1.25->5 40% 5->20 20% +{% set quarterly_cbm_rank_list = [1.75, 10] %} +{% set yearly_cbm_rank_list = [2.5, 20] %} -- IMPORT WITH shipping_companies AS ( @@ -99,10 +99,8 @@ dates AS ( THEN 'LOW_USAGE' WHEN (total_cbm_completed_approved > {{cbm_rank_list[0]}} AND total_cbm_completed_approved <= {{cbm_rank_list[1]}}) THEN 'MEDIUM_USAGE' - WHEN (total_cbm_completed_approved > {{cbm_rank_list[1]}} AND total_cbm_completed_approved <= {{cbm_rank_list[2]}}) + WHEN (total_cbm_completed_approved > {{cbm_rank_list[1]}}) THEN 'HIGH_USAGE' - WHEN total_cbm_completed_approved > {{cbm_rank_list[2]}} - THEN 'SUPER_HIGH_USAGE' ELSE 'ERROR' END AS company_usage_type,