diff --git a/models/intermediate/int_shipping__containers_group_by_packing_list_ids.sql b/models/intermediate/int_shipping__containers_group_by_packing_list_ids.sql deleted file mode 100644 index ee93c74..0000000 --- a/models/intermediate/int_shipping__containers_group_by_packing_list_ids.sql +++ /dev/null @@ -1,114 +0,0 @@ --- IMPORT -WITH containers AS ( - SELECT * FROM {{ ref('stg_shipping__containers') }} -), - -containers_brg_packing_lists AS ( - SELECT * FROM {{ ref('stg_shipping__containers_brg_packing_lists') }} -), - -container_transport_arrangments AS ( - SELECT * FROM {{ ref('stg_shipping__container_transport_arrangments') }} - -- WHERE deleted_datetime IS NULL -), - --- LOGIC -container_get_packing_list_ids AS ( - SELECT - containers_brg_packing_lists.packing_list_id, - containers.* - - FROM containers - - LEFT JOIN containers_brg_packing_lists - ON (containers.container_id = containers_brg_packing_lists.container_id) -), - -contianer_get_transport_arrangments AS ( - SELECT - container_get_packing_list_ids.packing_list_id, - container_get_packing_list_ids.container_id, - container_get_packing_list_ids.sub_company_id AS container_warehouse_sub_company_id, - container_get_packing_list_ids.container_reference, - container_get_packing_list_ids.container_number, - container_get_packing_list_ids.container_type, - container_get_packing_list_ids.seal_reference AS container_seal_reference, - container_get_packing_list_ids.status AS container_status, - container_get_packing_list_ids.loaded_datetime AS container_loaded_datetime, - container_get_packing_list_ids.deleted_datetime AS container_deleted_datetime, - container_get_packing_list_ids.created_datetime AS container_created_datetime, - container_get_packing_list_ids.updated_datetime AS container_updated_datetime, - container_transport_arrangments.transport_arrangement_id AS container_transport_arrangement_id, - container_transport_arrangments.dispatched_datetime AS container_departed_datetime, - container_transport_arrangments.droped_datetime AS container_arrived_datetime, - container_transport_arrangments.courier AS container_deliver_courier, - container_transport_arrangments.tracking_number AS container_tracking_number, - container_transport_arrangments.status AS container_transport_arrangment_status, - container_transport_arrangments.number_of_etd_datetime_postpone AS number_of_container_etd_datetime_postpone, - container_transport_arrangments.number_of_eta_datetime_postpone AS number_of_container_eta_datetime_postpone, - container_transport_arrangments.etd_datetimes AS container_etd_datetimes, - container_transport_arrangments.first_etd_datetime AS first_container_etd_datetimes, - container_transport_arrangments.last_etd_datetime AS last_container_etd_datetimes, - container_transport_arrangments.eta_datetimes AS container_eta_datetimes, - container_transport_arrangments.first_eta_datetime AS first_container_eta_datetime, - container_transport_arrangments.last_eta_datetime AS last_container_eta_datetime, - container_transport_arrangments.first_estimate_schedule_created_datetime AS first_container_estimate_schedule_created_datetime, - container_transport_arrangments.last_estimate_schedule_created_datetime AS last_container_estimate_schedule_created_datetime, - - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - - FROM container_get_packing_list_ids - - LEFT JOIN container_transport_arrangments - ON (container_get_packing_list_ids.container_id = container_transport_arrangments.container_id) -), - --- FINAL -final__int_shipping__containers_group_by_packing_list_ids AS ( - SELECT - -- ids - packing_list_id, - container_id, - container_warehouse_sub_company_id, - container_transport_arrangement_id, - - -- dimensions - container_reference, - container_number, - container_seal_reference, - container_type, - container_status, - container_transport_arrangment_status, - container_deliver_courier, - container_tracking_number, - - - -- measures - number_of_container_etd_datetime_postpone, - number_of_container_eta_datetime_postpone, - - -- date/times - container_loaded_datetime, - container_departed_datetime, - container_arrived_datetime, - container_etd_datetimes, - first_container_etd_datetimes, - last_container_etd_datetimes, - container_eta_datetimes, - first_container_eta_datetime, - last_container_eta_datetime, - first_container_estimate_schedule_created_datetime, - last_container_estimate_schedule_created_datetime, - container_deleted_datetime, - container_created_datetime, - container_updated_datetime, - - -- metadata - _dbt_ran_datetime - - FROM contianer_get_transport_arrangments -) --- -- YAM already do the hardcode clean, but not validate by omair --- SELECT * FROM container_group_by_packing_list_id WHERE packing_list_id in (21276, 16087) OR packing_list_id IS NULL - -SELECT * FROM final__int_shipping__containers_group_by_packing_list_ids \ No newline at end of file diff --git a/models/intermediate/int_shipping__destination_warehouse_last_mile_transport_arrangements.sql b/models/intermediate/int_shipping__destination_warehouse_last_mile_transport_arrangements.sql new file mode 100644 index 0000000..7fbd013 --- /dev/null +++ b/models/intermediate/int_shipping__destination_warehouse_last_mile_transport_arrangements.sql @@ -0,0 +1,122 @@ +-- IMPORT +WITH warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__warehouse_shipping_packing_lists') }} +), + +packing_list_transport_arrangments AS ( + SELECT * FROM {{ ref('stg_shipping__packing_list_transport_arrangments') }} +), + + +-- LOGIC +warehouse_shipping_join_transport_arrangements AS ( + + SELECT + warehouse_shipping_packing_lists.new_warehouse_shipping_packing_list_id, + warehouse_shipping_packing_lists.old_warehouse_shipping_packing_list_id, + warehouse_shipping_packing_lists.warehouse_order_number, + warehouse_shipping_packing_lists.booking_id, + warehouse_shipping_packing_lists.warehouse_sub_company_id, + warehouse_shipping_packing_lists.has_replaced_by_new_id, + warehouse_shipping_packing_lists.packing_list_type, + warehouse_shipping_packing_lists.reference_contract AS destination_warehouse_reference_contract, + warehouse_shipping_packing_lists.status AS destination_warehouse_shipping_status, + warehouse_shipping_packing_lists.deleted_datetime AS destination_warehouse_shipping_deleted_datetime, + + packing_list_transport_arrangments.transport_arrangement_id AS destination_warehouse_transport_arrangement_id, + packing_list_transport_arrangments.transport_mode AS destination_warehouse_transport_transport_mode, + packing_list_transport_arrangments.courier AS destination_warehouse_transport_courier, + packing_list_transport_arrangments.tracking_number AS destination_warehouse_transport_tracking_number, + packing_list_transport_arrangments.status AS destination_warehouse_transport_status, + packing_list_transport_arrangments.number_of_etd_datetime_postpone AS destination_warehouse_transport_number_of_etd_datetime_postpone, + packing_list_transport_arrangments.number_of_eta_datetime_postpone AS destination_warehouse_transport_number_of_eta_datetime_postpone, + packing_list_transport_arrangments.dispatched_datetime AS destination_warehouse_transport_dispatched_datetime, + packing_list_transport_arrangments.droped_datetime AS destination_warehouse_transport_droped_datetime, + packing_list_transport_arrangments.etd_datetimes AS destination_warehouse_transport_etd_datetimes, + packing_list_transport_arrangments.first_etd_datetime AS destination_warehouse_transport_first_etd_datetime, + packing_list_transport_arrangments.last_etd_datetime AS destination_warehouse_transport_last_etd_datetime, + packing_list_transport_arrangments.eta_datetimes AS destination_warehouse_transport_eta_datetimes, + packing_list_transport_arrangments.first_eta_datetime AS destination_warehouse_transport_first_eta_datetime, + packing_list_transport_arrangments.last_eta_datetime AS destination_warehouse_transport_last_eta_datetime, + packing_list_transport_arrangments.first_estimate_schedule_created_datetime AS destination_warehouse_transport_first_estimate_schedule_created_datetime, + packing_list_transport_arrangments.last_estimate_schedule_created_datetime AS destination_warehouse_transport_last_estimate_schedule_created_datetime, + packing_list_transport_arrangments.created_datetime AS destination_warehouse_transport_created_datetime, + packing_list_transport_arrangments.updated_datetime AS destination_warehouse_transport_updated_datetime + + FROM warehouse_shipping_packing_lists + + LEFT JOIN packing_list_transport_arrangments + ON ( + warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = packing_list_transport_arrangments.packing_list_id + OR + warehouse_shipping_packing_lists.new_origin_warehouse_shipping_packing_list_id = packing_list_transport_arrangments.packing_list_id + ) + +), + +get_lastest_created_row_only AS ( --Because the dataset not accurate, many duplicate withough deleted all those duplicate row + + SELECT + *, + ROW_NUMBER() OVER (PARTITION BY old_origin_warehouse_shipping_packing_list_id ORDER BY destination_warehouse_transport_created_datetime DESC) AS row_number_index, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM warehouse_shipping_join_transport_arrangements + + QUALIFY + row_number_index = 1 + +), + + +-- FINAL +final__int_shipping__destination_warehouse_last_mile_transport_arrangements AS ( + + SELECT + -- ids + new_origin_warehouse_shipping_packing_list_id, + old_origin_warehouse_shipping_packing_list_id, + destination_warehouse_transport_arrangement_id, + destination_warehouse_transport_arrangement_id, + booking_id, + warehouse_sub_company_id, + warehouse_order_number, + + -- dimensions + has_replaced_by_new_id, + reference_contract, + packing_list_type, + destination_warehouse_shipping_status, + destination_warehouse_transport_transport_mode, + destination_warehouse_transport_courier, + destination_warehouse_transport_tracking_number, + destination_warehouse_transport_status, + + -- measures + destination_warehouse_transport_number_of_etd_datetime_postpone, + destination_warehouse_transport_number_of_eta_datetime_postpone, + + -- date/times + destination_warehouse_transport_dispatched_datetime, + destination_warehouse_transport_droped_datetime, + destination_warehouse_transport_etd_datetimes, + destination_warehouse_transport_first_etd_datetime, + destination_warehouse_transport_last_etd_datetime, + destination_warehouse_transport_eta_datetimes, + destination_warehouse_transport_first_eta_datetime, + destination_warehouse_transport_last_eta_datetime, + destination_warehouse_transport_first_estimate_schedule_created_datetime, + destination_warehouse_transport_last_estimate_schedule_created_datetime, + destination_warehouse_transport_created_datetime, + destination_warehouse_transport_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM get_lastest_created_row_only + +) + +SELECT * FROM final__int_shipping__destination_warehouse_last_mile_transport_arrangements +-- Missing information on last mile delivery due to supplier not giving info. \ No newline at end of file diff --git a/models/intermediate/int_shipping__destination_warehouse_packing_list_transport_arrangements.sql b/models/intermediate/int_shipping__destination_warehouse_packing_list_transport_arrangements.sql deleted file mode 100644 index 3fef753..0000000 --- a/models/intermediate/int_shipping__destination_warehouse_packing_list_transport_arrangements.sql +++ /dev/null @@ -1,90 +0,0 @@ --- IMPORT -WITH origin_warehouse_shipping_packing_lists AS ( - SELECT * FROM {{ ref('stg_shipping__origin_warehouse_shipping_packing_lists') }} -), - -packing_list_transport_arrangments AS ( - SELECT * FROM {{ ref('stg_shipping__packing_list_transport_arrangments') }} -), - - --- LOGIC -remove_non_destination_warehouse_transport_arrangments AS ( - SELECT - origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id, - packing_list_transport_arrangments.transport_arrangement_id AS destination_warehouse_transport_arrangement_id, - packing_list_transport_arrangments.transport_mode AS destination_warehouse_transport_transport_mode, - packing_list_transport_arrangments.courier AS destination_warehouse_transport_courier, - packing_list_transport_arrangments.tracking_number AS destination_warehouse_transport_tracking_number, - packing_list_transport_arrangments.status AS destination_warehouse_transport_status, - packing_list_transport_arrangments.number_of_etd_datetime_postpone AS destination_warehouse_transport_number_of_etd_datetime_postpone, - packing_list_transport_arrangments.number_of_eta_datetime_postpone AS destination_warehouse_transport_number_of_eta_datetime_postpone, - packing_list_transport_arrangments.dispatched_datetime AS destination_warehouse_transport_dispatched_datetime, - packing_list_transport_arrangments.droped_datetime AS destination_warehouse_transport_droped_datetime, - packing_list_transport_arrangments.etd_datetimes AS destination_warehouse_transport_etd_datetimes, - packing_list_transport_arrangments.first_etd_datetime AS destination_warehouse_transport_first_etd_datetime, - packing_list_transport_arrangments.last_etd_datetime AS destination_warehouse_transport_last_etd_datetime, - packing_list_transport_arrangments.eta_datetimes AS destination_warehouse_transport_eta_datetimes, - packing_list_transport_arrangments.first_eta_datetime AS destination_warehouse_transport_first_eta_datetime, - packing_list_transport_arrangments.last_eta_datetime AS destination_warehouse_transport_last_eta_datetime, - packing_list_transport_arrangments.first_estimate_schedule_created_datetime AS destination_warehouse_transport_first_estimate_schedule_created_datetime, - packing_list_transport_arrangments.last_estimate_schedule_created_datetime AS destination_warehouse_transport_last_estimate_schedule_created_datetime, - packing_list_transport_arrangments.created_datetime AS destination_warehouse_transport_created_datetime, - packing_list_transport_arrangments.updated_datetime AS destination_warehouse_transport_updated_datetime - - FROM origin_warehouse_shipping_packing_lists - - INNER JOIN packing_list_transport_arrangments - ON (origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = packing_list_transport_arrangments.packing_list_id) -), - -get_lastest_created_row_only AS ( --Because the dataset not accurate, many duplicate withough deleted all those duplicate row - SELECT - *, - ROW_NUMBER() OVER (PARTITION BY old_origin_warehouse_shipping_packing_list_id ORDER BY destination_warehouse_transport_created_datetime DESC) AS row_number_index, - - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - - FROM remove_non_destination_warehouse_transport_arrangments - - QUALIFY row_number_index = 1 -), - --- FINAL -final__int_shipping__destination_warehouse_packing_list_transport_arrangements AS ( - SELECT - -- ids - destination_warehouse_transport_arrangement_id, - old_origin_warehouse_shipping_packing_list_id, - - -- dimensions - destination_warehouse_transport_transport_mode, - destination_warehouse_transport_courier, - destination_warehouse_transport_tracking_number, - destination_warehouse_transport_status, - - -- measures - destination_warehouse_transport_number_of_etd_datetime_postpone, - destination_warehouse_transport_number_of_eta_datetime_postpone, - - -- date/times - destination_warehouse_transport_dispatched_datetime, - destination_warehouse_transport_droped_datetime, - destination_warehouse_transport_etd_datetimes, - destination_warehouse_transport_first_etd_datetime, - destination_warehouse_transport_last_etd_datetime, - destination_warehouse_transport_eta_datetimes, - destination_warehouse_transport_first_eta_datetime, - destination_warehouse_transport_last_eta_datetime, - destination_warehouse_transport_first_estimate_schedule_created_datetime, - destination_warehouse_transport_last_estimate_schedule_created_datetime, - destination_warehouse_transport_created_datetime, - destination_warehouse_transport_updated_datetime, - - -- metadata - _dbt_ran_datetime - - FROM get_lastest_created_row_only -) - -SELECT * FROM final__int_shipping__destination_warehouse_packing_list_transport_arrangements \ No newline at end of file diff --git a/models/intermediate/int_shipping__origin_factory_packing_list_transport_arrangements.sql b/models/intermediate/int_shipping__origin_factory_packing_list_transport_arrangements.sql deleted file mode 100644 index 561879e..0000000 --- a/models/intermediate/int_shipping__origin_factory_packing_list_transport_arrangements.sql +++ /dev/null @@ -1,81 +0,0 @@ --- IMPORT -WITH origin_warehouse_receive_packing_lists AS ( - SELECT * FROM {{ ref('stg_shipping__origin_warehouse_receive_packing_lists') }} -), - -packing_list_transport_arrangments AS ( - SELECT * FROM {{ ref('stg_shipping__packing_list_transport_arrangments') }} -), - - --- LOGIC -remove_non_origin_factory_transport_arrangments AS ( - SELECT - origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id, - packing_list_transport_arrangments.transport_arrangement_id AS origin_factory_transport_arrangement_id, - packing_list_transport_arrangments.transport_mode AS origin_factory_transport_transport_mode, - packing_list_transport_arrangments.courier AS origin_factory_transport_courier, - packing_list_transport_arrangments.tracking_number AS origin_factory_transport_tracking_number, - packing_list_transport_arrangments.status AS origin_factory_transport_status, - packing_list_transport_arrangments.number_of_etd_datetime_postpone AS origin_factory_transport_number_of_etd_datetime_postpone, - packing_list_transport_arrangments.number_of_eta_datetime_postpone AS origin_factory_transport_number_of_eta_datetime_postpone, - packing_list_transport_arrangments.dispatched_datetime AS origin_factory_transport_dispatched_datetime, - packing_list_transport_arrangments.droped_datetime AS origin_factory_transport_droped_datetime, - packing_list_transport_arrangments.etd_datetimes AS origin_factory_transport_etd_datetimes, - packing_list_transport_arrangments.first_etd_datetime AS origin_factory_transport_first_etd_datetime, - packing_list_transport_arrangments.last_etd_datetime AS origin_factory_transport_last_etd_datetime, - packing_list_transport_arrangments.eta_datetimes AS origin_factory_transport_eta_datetimes, - packing_list_transport_arrangments.first_eta_datetime AS origin_factory_transport_first_eta_datetime, - packing_list_transport_arrangments.last_eta_datetime AS origin_factory_transport_last_eta_datetime, - packing_list_transport_arrangments.first_estimate_schedule_created_datetime AS origin_factory_transport_first_estimate_schedule_created_datetime, - packing_list_transport_arrangments.last_estimate_schedule_created_datetime AS origin_factory_transport_last_estimate_schedule_created_datetime, - packing_list_transport_arrangments.created_datetime AS origin_factory_transport_created_datetime, - packing_list_transport_arrangments.updated_datetime AS origin_factory_transport_updated_datetime, - - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - - FROM origin_warehouse_receive_packing_lists - - INNER JOIN packing_list_transport_arrangments - ON (origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id = packing_list_transport_arrangments.packing_list_id) -), - --- FINAL -final__int_shipping__origin_factory_packing_list_transport_arrangements AS ( - SELECT - -- ids - origin_factory_transport_arrangement_id, - old_origin_warehouse_receive_packing_list_id, - - -- dimensions - origin_factory_transport_transport_mode, - origin_factory_transport_courier, - origin_factory_transport_tracking_number, - origin_factory_transport_status, - - -- measures - origin_factory_transport_number_of_etd_datetime_postpone, - origin_factory_transport_number_of_eta_datetime_postpone, - - -- date/times - origin_factory_transport_dispatched_datetime, - origin_factory_transport_droped_datetime, - origin_factory_transport_etd_datetimes, - origin_factory_transport_first_etd_datetime, - origin_factory_transport_last_etd_datetime, - origin_factory_transport_eta_datetimes, - origin_factory_transport_first_eta_datetime, - origin_factory_transport_last_eta_datetime, - origin_factory_transport_first_estimate_schedule_created_datetime, - origin_factory_transport_last_estimate_schedule_created_datetime, - - origin_factory_transport_created_datetime, - origin_factory_transport_updated_datetime, - - -- metadata - _dbt_ran_datetime - - FROM remove_non_origin_factory_transport_arrangments -) - -SELECT * FROM final__int_shipping__origin_factory_packing_list_transport_arrangements \ No newline at end of file diff --git a/models/intermediate/int_shipping__origin_warehouse_container_shipping_transport_arrangements.sql b/models/intermediate/int_shipping__origin_warehouse_container_shipping_transport_arrangements.sql new file mode 100644 index 0000000..a227da8 --- /dev/null +++ b/models/intermediate/int_shipping__origin_warehouse_container_shipping_transport_arrangements.sql @@ -0,0 +1,150 @@ +-- IMPORT +WITH origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__warehouse_shipping_packing_lists') }} +), + +containers AS ( + SELECT * FROM {{ ref('stg_shipping__containers') }} +), + +containers_brg_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__containers_brg_packing_lists') }} +), + +container_transport_arrangments AS ( + SELECT * FROM {{ ref('stg_shipping__container_transport_arrangments') }} +), + + +-- LOGIC +container_get_packing_list_ids AS ( + + SELECT + containers_brg_packing_lists.packing_list_id, + containers.* + + FROM containers + + LEFT JOIN containers_brg_packing_lists + ON (containers.container_id = containers_brg_packing_lists.container_id) + +), + +container_get_transport_arrangments AS ( + + SELECT + origin_warehouse_shipping_packing_lists.new_origin_warehouse_shipping_packing_list_id, + origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id, + origin_warehouse_shipping_packing_lists.warehouse_order_number, + origin_warehouse_shipping_packing_lists.booking_id, + origin_warehouse_shipping_packing_lists.warehouse_sub_company_id, + origin_warehouse_shipping_packing_lists.has_replaced_by_new_id, + origin_warehouse_shipping_packing_lists.reference_contract AS origin_warehouse_reference_contract, + origin_warehouse_shipping_packing_lists.status AS origin_warehouse_shipping_status, + origin_warehouse_shipping_packing_lists.packing_list_type, + + container_get_packing_list_ids.packing_list_id AS container_packing_list_id, + container_get_packing_list_ids.container_id, + container_get_packing_list_ids.sub_company_id AS container_warehouse_sub_company_id, + container_get_packing_list_ids.container_reference, + container_get_packing_list_ids.container_number, + container_get_packing_list_ids.container_type, + container_get_packing_list_ids.seal_reference AS container_seal_reference, + container_get_packing_list_ids.status AS container_status, + container_get_packing_list_ids.loaded_datetime AS container_loaded_datetime, + container_get_packing_list_ids.deleted_datetime AS container_deleted_datetime, + container_get_packing_list_ids.created_datetime AS container_created_datetime, + container_get_packing_list_ids.updated_datetime AS container_updated_datetime, + + container_transport_arrangments.transport_arrangement_id AS container_transport_arrangement_id, + container_transport_arrangments.dispatched_datetime AS container_departed_datetime, + container_transport_arrangments.droped_datetime AS container_arrived_datetime, + container_transport_arrangments.courier AS container_deliver_courier, + container_transport_arrangments.tracking_number AS container_tracking_number, + container_transport_arrangments.status AS container_transport_arrangment_status, + container_transport_arrangments.number_of_etd_datetime_postpone AS number_of_container_etd_datetime_postpone, + container_transport_arrangments.number_of_eta_datetime_postpone AS number_of_container_eta_datetime_postpone, + container_transport_arrangments.etd_datetimes AS container_etd_datetimes, + container_transport_arrangments.first_etd_datetime AS first_container_etd_datetimes, + container_transport_arrangments.last_etd_datetime AS last_container_etd_datetimes, + container_transport_arrangments.eta_datetimes AS container_eta_datetimes, + container_transport_arrangments.first_eta_datetime AS first_container_eta_datetime, + container_transport_arrangments.last_eta_datetime AS last_container_eta_datetime, + container_transport_arrangments.first_estimate_schedule_created_datetime AS first_container_estimate_schedule_created_datetime, + container_transport_arrangments.last_estimate_schedule_created_datetime AS last_container_estimate_schedule_created_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN container_get_packing_list_ids + ON ( + origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = container_get_packing_list_ids.packing_list_id + OR + origin_warehouse_shipping_packing_lists.new_origin_warehouse_shipping_packing_list_id = container_get_packing_list_ids.packing_list_id + ) + + LEFT JOIN container_transport_arrangments + ON (container_get_packing_list_ids.container_id = container_transport_arrangments.container_id) + +), + + +-- FINAL +final__int_shipping__containers_group_by_packing_list_ids AS ( + + SELECT + -- ids + new_origin_warehouse_shipping_packing_list_id, + old_origin_warehouse_shipping_packing_list_id, + container_transport_arrangement_id, + booking_id, + container_id, + container_warehouse_sub_company_id, + warehouse_sub_company_id, + warehouse_order_number, + + -- dimensions + has_replaced_by_new_id, + origin_warehouse_shipping_status, + origin_warehouse_reference_contract, + packing_list_type, + container_reference, + container_number, + container_seal_reference, + container_type, + container_status, + container_transport_arrangment_status, + container_deliver_courier, + container_tracking_number, + + -- measures + number_of_container_etd_datetime_postpone, + number_of_container_eta_datetime_postpone, + + -- date/times + container_loaded_datetime, + container_departed_datetime, + container_arrived_datetime, + container_etd_datetimes, + first_container_etd_datetimes, + last_container_etd_datetimes, + container_eta_datetimes, + first_container_eta_datetime, + last_container_eta_datetime, + first_container_estimate_schedule_created_datetime, + last_container_estimate_schedule_created_datetime, + container_deleted_datetime, + container_created_datetime, + container_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM container_get_transport_arrangments + +) +-- -- YAM already do the hardcode clean, but not validate by omair +-- SELECT * FROM container_group_by_packing_list_id WHERE packing_list_id in (21276, 16087) OR packing_list_id IS NULL + +SELECT * FROM final__int_shipping__containers_group_by_packing_list_ids \ No newline at end of file diff --git a/models/intermediate/int_shipping__origin_warehouse_receive_transport_arrangements.sql b/models/intermediate/int_shipping__origin_warehouse_receive_transport_arrangements.sql new file mode 100644 index 0000000..21392da --- /dev/null +++ b/models/intermediate/int_shipping__origin_warehouse_receive_transport_arrangements.sql @@ -0,0 +1,101 @@ +-- IMPORT +WITH origin_warehouse_receive_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__origin_warehouse_receive_packing_lists') }} +), + +packing_list_transport_arrangments AS ( + SELECT * FROM {{ ref('stg_shipping__packing_list_transport_arrangments') }} +), + + +-- LOGIC +origin_factory_packing_list_join_transport_arrangments AS ( + + SELECT + origin_warehouse_receive_packing_lists.new_origin_warehouse_receive_packing_list_id, + origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id, + origin_warehouse_receive_packing_lists.warehouse_order_number, + origin_warehouse_receive_packing_lists.booking_id, + origin_warehouse_receive_packing_lists.warehouse_sub_company_id, + origin_warehouse_receive_packing_lists.has_replaced_by_new_id, + origin_warehouse_receive_packing_lists.reference_contract, + origin_warehouse_receive_packing_lists.status AS origin_warehouse_receive_status, + + packing_list_transport_arrangments.transport_arrangement_id AS origin_factory_transport_arrangement_id, + packing_list_transport_arrangments.transport_mode AS origin_factory_transport_transport_mode, + packing_list_transport_arrangments.courier AS origin_factory_transport_courier, + packing_list_transport_arrangments.tracking_number AS origin_factory_transport_tracking_number, + packing_list_transport_arrangments.status AS origin_factory_transport_status, + packing_list_transport_arrangments.number_of_etd_datetime_postpone AS origin_factory_transport_number_of_etd_datetime_postpone, + packing_list_transport_arrangments.number_of_eta_datetime_postpone AS origin_factory_transport_number_of_eta_datetime_postpone, + packing_list_transport_arrangments.dispatched_datetime AS origin_factory_dispatched_datetime, + packing_list_transport_arrangments.droped_datetime AS origin_warehouse_received_datetime, + packing_list_transport_arrangments.etd_datetimes AS origin_factory_transport_etd_datetimes, + packing_list_transport_arrangments.first_etd_datetime AS origin_factory_transport_first_etd_datetime, + packing_list_transport_arrangments.last_etd_datetime AS origin_factory_transport_last_etd_datetime, + packing_list_transport_arrangments.eta_datetimes AS origin_factory_transport_eta_datetimes, + packing_list_transport_arrangments.first_eta_datetime AS origin_factory_transport_first_eta_datetime, + packing_list_transport_arrangments.last_eta_datetime AS origin_factory_transport_last_eta_datetime, + packing_list_transport_arrangments.first_estimate_schedule_created_datetime AS origin_factory_transport_first_estimate_schedule_created_datetime, + packing_list_transport_arrangments.last_estimate_schedule_created_datetime AS origin_factory_transport_last_estimate_schedule_created_datetime, + packing_list_transport_arrangments.created_datetime AS origin_factory_transport_created_datetime, + packing_list_transport_arrangments.updated_datetime AS origin_factory_transport_updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM origin_warehouse_receive_packing_lists + + LEFT JOIN packing_list_transport_arrangments + ON (origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id = packing_list_transport_arrangments.packing_list_id) + +), + + +-- FINAL +final__int_shipping__origin_factory_packing_list_transport_arrangements AS ( + + SELECT + -- ids + new_origin_warehouse_receive_packing_list_id, + old_origin_warehouse_receive_packing_list_id, + origin_factory_transport_arrangement_id, + booking_id, + warehouse_order_number, + warehouse_sub_company_id, + + -- dimensions + has_replaced_by_new_id, + reference_contract, + origin_factory_transport_transport_mode, + origin_factory_transport_courier, + origin_factory_transport_tracking_number, + origin_factory_transport_status, + origin_warehouse_receive_status, + + -- measures + origin_factory_transport_number_of_etd_datetime_postpone, + origin_factory_transport_number_of_eta_datetime_postpone, + + -- date/times + origin_factory_dispatched_datetime, + origin_warehouse_received_datetime, + origin_factory_transport_etd_datetimes, + origin_factory_transport_first_etd_datetime, + origin_factory_transport_last_etd_datetime, + origin_factory_transport_eta_datetimes, + origin_factory_transport_first_eta_datetime, + origin_factory_transport_last_eta_datetime, + origin_factory_transport_first_estimate_schedule_created_datetime, + origin_factory_transport_last_estimate_schedule_created_datetime, + + origin_factory_transport_created_datetime, + origin_factory_transport_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM origin_factory_packing_list_join_transport_arrangments + +) + +SELECT * FROM final__int_shipping__origin_factory_packing_list_transport_arrangements \ No newline at end of file diff --git a/models/intermediate/int_shipping__vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.sql b/models/intermediate/int_shipping__vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.sql index eddedf3..abc392d 100644 --- a/models/intermediate/int_shipping__vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.sql +++ b/models/intermediate/int_shipping__vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.sql @@ -4,7 +4,7 @@ WITH origin_warehouse_receive_packing_lists AS ( ), origin_warehouse_shipping_packing_lists AS ( - SELECT * FROM {{ ref('stg_shipping__origin_warehouse_shipping_packing_lists') }} + SELECT * FROM {{ ref('stg_shipping__warehouse_shipping_packing_lists') }} ), diff --git a/models/marts/warehouse/fct_shipping__origin_warehouse_receive_packing_lists.sql b/models/marts/warehouse/fct_shipping__origin_warehouse_receive_packing_lists.sql index 891b001..3e1cdbe 100644 --- a/models/marts/warehouse/fct_shipping__origin_warehouse_receive_packing_lists.sql +++ b/models/marts/warehouse/fct_shipping__origin_warehouse_receive_packing_lists.sql @@ -4,7 +4,7 @@ WITH origin_warehouse_receive_packing_lists AS ( ), origin_warehouse_shipping_packing_lists AS ( - SELECT * FROM {{ ref('stg_shipping__origin_warehouse_shipping_packing_lists') }} + SELECT * FROM {{ ref('stg_shipping__warehouse_shipping_packing_lists') }} ), vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists AS ( @@ -19,12 +19,12 @@ booking_delivery_and_billing_addresses AS ( SELECT * FROM {{ ref('int_shipping__booking_delivery_and_billing_addresses') }} ), -origin_factory_packing_list_transport_arrangements AS ( - SELECT * FROM {{ ref('int_shipping__origin_factory_packing_list_transport_arrangements') }} +origin_warehouse_receive_brg_transport_arrangements AS ( + SELECT * FROM {{ ref('int_shipping__origin_warehouse_receive_transport_arrangements') }} ), destination_warehouse_packing_list_transport_arrangements AS ( - SELECT * FROM {{ ref('int_shipping__destination_warehouse_packing_list_transport_arrangements') }} + SELECT * FROM {{ ref('int_shipping__destination_warehouse_last_mile_transport_arrangements') }} ), packing_list_invoice_payments_group_by_packing_list_ids AS ( @@ -36,7 +36,7 @@ packages_group_by_packing_list_ids AS ( ), container_group_by_packing_list_ids AS ( - SELECT * FROM {{ ref('int_shipping__containers_group_by_packing_list_ids') }} + SELECT * FROM {{ ref('int_shipping__origin_warehouse_container_shipping_transport_arrangements') }} ), @@ -100,24 +100,24 @@ origin_warehouse_receive_packing_lists_enrich AS ( packages_group_by_packing_list_ids.number_of_pallet_charge_record, packages_group_by_packing_list_ids.pallet_charge_cbm, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_arrangement_id, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_transport_mode, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_courier, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_tracking_number, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_status, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_number_of_etd_datetime_postpone, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_number_of_eta_datetime_postpone, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_dispatched_datetime, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_droped_datetime, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_etd_datetimes, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_first_etd_datetime, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_last_etd_datetime, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_eta_datetimes, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_first_eta_datetime, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_last_eta_datetime, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_first_estimate_schedule_created_datetime, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_last_estimate_schedule_created_datetime, - origin_factory_packing_list_transport_arrangements.origin_factory_transport_created_datetime + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_arrangement_id, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_transport_mode, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_courier, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_tracking_number, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_status, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_number_of_etd_datetime_postpone, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_number_of_eta_datetime_postpone, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_dispatched_datetime, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_droped_datetime, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_etd_datetimes, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_first_etd_datetime, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_last_etd_datetime, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_eta_datetimes, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_first_eta_datetime, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_last_eta_datetime, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_first_estimate_schedule_created_datetime, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_last_estimate_schedule_created_datetime, + origin_warehouse_receive_brg_transport_arrangements.origin_factory_transport_created_datetime FROM origin_warehouse_receive_packing_lists @@ -133,8 +133,8 @@ origin_warehouse_receive_packing_lists_enrich AS ( LEFT JOIN packages_group_by_packing_list_ids ON (origin_warehouse_receive_packing_lists.origin_warehouse_receive_packing_list_id = packages_group_by_packing_list_ids.packing_list_id) - LEFT JOIN origin_factory_packing_list_transport_arrangements - ON (origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id = origin_factory_packing_list_transport_arrangements.old_origin_warehouse_receive_packing_list_id) + LEFT JOIN origin_warehouse_receive_brg_transport_arrangements + ON (origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id = origin_warehouse_receive_brg_transport_arrangements.old_origin_warehouse_receive_packing_list_id) ), origin_warehouse_shipping_packing_lists_enrich AS ( diff --git a/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql b/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql index f5cc8f0..2a1fd99 100644 --- a/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql +++ b/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql @@ -1,6 +1,6 @@ -- IMPORT WITH origin_warehouse_shipping_packing_lists AS ( - SELECT * FROM {{ ref('stg_shipping__origin_warehouse_shipping_packing_lists') }} + SELECT * FROM {{ ref('stg_shipping__warehouse_shipping_packing_lists') }} ), bookings AS ( @@ -12,7 +12,7 @@ booking_delivery_and_billing_addresses AS ( ), destination_warehouse_packing_list_transport_arrangements AS ( - SELECT * FROM {{ ref('int_shipping__destination_warehouse_packing_list_transport_arrangements') }} + SELECT * FROM {{ ref('int_shipping__destination_warehouse_last_mile_transport_arrangements') }} ), packing_list_invoice_payments_group_by_packing_list_ids AS ( @@ -24,7 +24,7 @@ packages_group_by_packing_list_ids AS ( ), container_group_by_packing_list_ids AS ( - SELECT * FROM {{ ref('int_shipping__containers_group_by_packing_list_ids') }} + SELECT * FROM {{ ref('int_shipping__origin_warehouse_container_shipping_transport_arrangements') }} ), diff --git a/models/staging/shipping/stg_shipping__containers_brg_packing_lists.sql b/models/staging/shipping/stg_shipping__containers_brg_packing_lists.sql index c798fa7..bb8fe20 100644 --- a/models/staging/shipping/stg_shipping__containers_brg_packing_lists.sql +++ b/models/staging/shipping/stg_shipping__containers_brg_packing_lists.sql @@ -8,28 +8,44 @@ WITH container_packing_lists AS ( container_packing_lists_rename AS ( SELECT - container_packing_lists.id AS container_packing_list_id, + container_packing_lists.id AS container_packing_list_id, container_packing_lists.container_id, container_packing_lists.packing_list_id, - container_packing_lists.deleted_at AS deleted_datetime, - container_packing_lists.created_at AS created_datetime, - container_packing_lists.updated_at AS updated_datetime + container_packing_lists.deleted_at AS deleted_datetime, + container_packing_lists.created_at AS created_datetime, + container_packing_lists.updated_at AS updated_datetime FROM container_packing_lists + ), -hardcode_remove_one_packing_list_brg_to_multiple_container AS ( +hard_code_remove_one_packing_list_brg_to_multiple_container AS ( + /* Data Anomaly (1) + Duplicated packing_list_id found IN (16087, 21276) + These packing_list_id are found in multiple container_id. + Hard code to remove the duplicated rows. + + Data Anomaly (2) + Duplicated packing_list_id found IN (30602, 30612, 30616, 30624) + These packing_list_id have same container_id but different primary key id. + Remove duplicates. + */ SELECT *, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + ROW_NUMBER() OVER (PARTITION BY packing_list_id ORDER BY container_id, container_packing_list_id) AS row_num_idx, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM container_packing_lists_rename - -- This 4 container_packing_list_id container multiple container in a pacing_list (172922,273744,336138,383558) - -- Remove the duplicate WHERE container_packing_list_id NOT IN (383558,273744) + + QUALIFY + row_num_idx = 1 + ), + -- FINAL final__stg_shipping__containers_brg_packing_lists AS ( @@ -39,10 +55,6 @@ final__stg_shipping__containers_brg_packing_lists AS ( container_id, packing_list_id, - -- dimensions - - -- measures - -- date/times deleted_datetime, created_datetime, @@ -51,7 +63,8 @@ final__stg_shipping__containers_brg_packing_lists AS ( -- metadata _dbt_ran_datetime - FROM hardcode_remove_one_packing_list_brg_to_multiple_container + FROM hard_code_remove_one_packing_list_brg_to_multiple_container + ) SELECT * FROM final__stg_shipping__containers_brg_packing_lists \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__origin_warehouse_receive_packing_lists.sql b/models/staging/shipping/stg_shipping__origin_warehouse_receive_packing_lists.sql index fcba2db..27071ae 100644 --- a/models/staging/shipping/stg_shipping__origin_warehouse_receive_packing_lists.sql +++ b/models/staging/shipping/stg_shipping__origin_warehouse_receive_packing_lists.sql @@ -13,8 +13,14 @@ status AS ( WHERE category = 'DEFAULT' ), +packages AS ( + SELECT * FROM {{ ref('base_shipping__packages') }} +), + + -- LOGIC packing_lists_type_mapping AS ( + SELECT packing_lists.packing_list_id, packing_lists.owner_id, @@ -22,8 +28,8 @@ packing_lists_type_mapping AS ( packing_lists.owner_type, packing_lists.reference_contract, packing_lists.reference, - COALESCE(packing_list_types.name, packing_lists.packing_list_type::string) AS packing_list_type, - COALESCE(status.name, packing_lists.status::string) AS status, + COALESCE(packing_list_types.name, packing_lists.packing_list_type::string) AS packing_list_type, + COALESCE(status.name, packing_lists.status::string) AS status, packing_lists.deleted_datetime, packing_lists.created_datetime, packing_lists.updated_datetime @@ -39,9 +45,10 @@ packing_lists_type_mapping AS ( ), warehouse_receive_lists AS ( + SELECT - packing_list_id, - owner_id AS booking_id, + packing_list_id AS original_packing_list_id, + owner_id AS booking_id, warehouse_sub_company_id, reference_contract, reference, @@ -59,12 +66,14 @@ warehouse_receive_lists AS ( owner_type = 'App\\Models\\Order' AND deleted_datetime IS NULL + ), warehouse_receive_list_replications AS ( + SELECT - packing_list_id AS new_packing_list_id, - owner_id AS packing_list_id, + packing_list_id AS new_packing_list_id, + owner_id AS original_packing_list_id, warehouse_sub_company_id, reference_contract, reference, @@ -81,48 +90,66 @@ warehouse_receive_list_replications AS ( owner_type = 'App\\Models\\PackingList' AND deleted_datetime IS NULL + ), warehouse_receive_list_replications_cover_warehouse_receive_lists AS ( - SELECT - COALESCE(warehouse_receive_list_replications.new_packing_list_id, warehouse_receive_lists.packing_list_id) AS origin_warehouse_receive_packing_list_id, + SELECT + COALESCE(warehouse_receive_list_replications.new_packing_list_id, warehouse_receive_lists.original_packing_list_id) AS new_origin_warehouse_receive_packing_list_id, + warehouse_receive_lists.original_packing_list_id AS old_origin_warehouse_receive_packing_list_id, warehouse_receive_lists.booking_id, warehouse_receive_lists.warehouse_sub_company_id, warehouse_receive_lists.reference_contract, - warehouse_receive_lists.reference AS warehouse_order_number, + warehouse_receive_lists.reference AS warehouse_order_number, warehouse_receive_lists.status, warehouse_receive_lists.deleted_datetime, warehouse_receive_lists.created_datetime, warehouse_receive_lists.updated_datetime, - warehouse_receive_lists.packing_list_id AS old_origin_warehouse_receive_packing_list_id, - IFF(warehouse_receive_list_replications.new_packing_list_id IS NULL, 0, 1) AS has_replaced_by_new_id, - - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + IFF(warehouse_receive_list_replications.new_packing_list_id IS NULL, 0, 1) AS has_replaced_by_new_id FROM warehouse_receive_lists LEFT JOIN warehouse_receive_list_replications - ON (warehouse_receive_lists.packing_list_id = warehouse_receive_list_replications.packing_list_id) + ON (warehouse_receive_lists.original_packing_list_id = warehouse_receive_list_replications.original_packing_list_id) + ), +warehouse_receive_list_clean AS ( + -- Packing list without packages are most likely error input by YD + SELECT + warehouse_receive_list_replications_cover_warehouse_receive_lists.*, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM warehouse_receive_list_replications_cover_warehouse_receive_lists + + WHERE + old_origin_warehouse_receive_packing_list_id IN ( SELECT DISTINCT packing_list_id FROM packages ) + AND + new_origin_warehouse_receive_packing_list_id IN ( SELECT DISTINCT packing_list_id FROM packages ) + +), + + -- FINAL final__stg_shipping__origin_warehouse_receive_packing_lists AS ( + SELECT -- ids - origin_warehouse_receive_packing_list_id, + new_origin_warehouse_receive_packing_list_id, old_origin_warehouse_receive_packing_list_id, - warehouse_order_number, booking_id, warehouse_sub_company_id, + warehouse_order_number, -- dimensions has_replaced_by_new_id, reference_contract, status, - -- measures + -- date/times deleted_datetime, created_datetime, updated_datetime, @@ -130,7 +157,8 @@ final__stg_shipping__origin_warehouse_receive_packing_lists AS ( -- metadata _dbt_ran_datetime - FROM warehouse_receive_list_replications_cover_warehouse_receive_lists + FROM warehouse_receive_list_clean + ) SELECT * FROM final__stg_shipping__origin_warehouse_receive_packing_lists \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__origin_warehouse_shipping_packing_lists.sql b/models/staging/shipping/stg_shipping__warehouse_shipping_packing_lists.sql similarity index 61% rename from models/staging/shipping/stg_shipping__origin_warehouse_shipping_packing_lists.sql rename to models/staging/shipping/stg_shipping__warehouse_shipping_packing_lists.sql index ce1fd24..88e36e5 100644 --- a/models/staging/shipping/stg_shipping__origin_warehouse_shipping_packing_lists.sql +++ b/models/staging/shipping/stg_shipping__warehouse_shipping_packing_lists.sql @@ -1,3 +1,10 @@ +/* +The shipping packing lists contain the shipment of goods in 2 different ways: +1) China warehouse shipping to Malaysia warehouse (By sea) +2) Malaysia warehouse shipping to customer (By land) +*/ + + -- IMPORT WITH packing_lists AS ( SELECT * FROM {{ ref('base_shipping__packing_lists') }} @@ -13,8 +20,14 @@ status AS ( WHERE category = 'DEFAULT' ), +packages AS ( + SELECT * FROM {{ ref('base_shipping__packages') }} +), + + -- LOGIC packing_lists_type_mapping AS ( + SELECT packing_lists.packing_list_id, packing_lists.owner_id, @@ -22,8 +35,8 @@ packing_lists_type_mapping AS ( packing_lists.owner_type, packing_lists.reference_contract, packing_lists.reference, - COALESCE(packing_list_types.name, packing_lists.packing_list_type::string) AS packing_list_type, - COALESCE(status.name, packing_lists.status::string) AS status, + COALESCE(packing_list_types.name, packing_lists.packing_list_type::string) AS packing_list_type, + COALESCE(status.name, packing_lists.status::string) AS status, packing_lists.deleted_datetime, packing_lists.created_datetime, packing_lists.updated_datetime @@ -39,12 +52,14 @@ packing_lists_type_mapping AS ( ), warehouse_shipping_lists AS ( - SELECT - packing_list_id, - owner_id AS booking_id, + + SELECT + packing_list_id AS original_packing_list_id, + owner_id AS booking_id, warehouse_sub_company_id, reference_contract, reference, + packing_list_type, status, deleted_datetime, created_datetime, @@ -59,15 +74,18 @@ warehouse_shipping_lists AS ( owner_type = 'App\\Models\\Order' AND deleted_datetime IS NULL + ), warehouse_shipping_list_replications AS ( + SELECT - packing_list_id AS new_packing_list_id, - owner_id AS packing_list_id, + packing_list_id AS new_packing_list_id, + owner_id AS original_packing_list_id, warehouse_sub_company_id, reference_contract, reference, + packing_list_type, deleted_datetime, created_datetime, updated_datetime @@ -81,45 +99,65 @@ warehouse_shipping_list_replications AS ( owner_type = 'App\\Models\\PackingList' AND deleted_datetime IS NULL + ), warehouse_shipping_list_replications_cover_warehouse_shipping_lists AS ( - SELECT - COALESCE(warehouse_shipping_list_replications.new_packing_list_id, warehouse_shipping_lists.packing_list_id) AS origin_warehouse_shipping_packing_list_id, + SELECT + COALESCE(warehouse_shipping_list_replications.new_packing_list_id, warehouse_shipping_lists.original_packing_list_id) AS new_warehouse_shipping_packing_list_id, + warehouse_shipping_lists.original_packing_list_id AS old_warehouse_shipping_packing_list_id, warehouse_shipping_lists.booking_id, warehouse_shipping_lists.warehouse_sub_company_id, warehouse_shipping_lists.reference_contract, - warehouse_shipping_lists.reference AS warehouse_order_number, + warehouse_shipping_lists.reference AS warehouse_order_number, + warehouse_shipping_lists.packing_list_type, warehouse_shipping_lists.status, warehouse_shipping_lists.deleted_datetime, warehouse_shipping_lists.created_datetime, warehouse_shipping_lists.updated_datetime, - warehouse_shipping_lists.packing_list_id AS old_origin_warehouse_shipping_packing_list_id, - IFF(warehouse_shipping_list_replications.new_packing_list_id IS NULL, 0, 1) AS has_replaced_by_new_id, - - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + IFF(warehouse_shipping_list_replications.new_packing_list_id IS NULL, 0, 1) AS has_replaced_by_new_id FROM warehouse_shipping_lists LEFT JOIN warehouse_shipping_list_replications - ON (warehouse_shipping_lists.packing_list_id = warehouse_shipping_list_replications.packing_list_id) + ON (warehouse_shipping_lists.original_packing_list_id = warehouse_shipping_list_replications.original_packing_list_id) + ), +warehouse_shipping_list_cleaned AS ( + -- Packing list without packages are most likely an error input. To filter out. + SELECT + warehouse_shipping_list_replications_cover_warehouse_shipping_lists.*, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM warehouse_shipping_list_replications_cover_warehouse_shipping_lists + + WHERE + new_warehouse_shipping_packing_list_id IN ( SELECT DISTINCT packing_list_id FROM packages ) + AND + new_warehouse_shipping_packing_list_id IN ( SELECT DISTINCT packing_list_id FROM packages ) + +), + + -- FINAL final__stg_shipping__origin_warehouse_shipping_packing_lists AS ( + SELECT -- ids - origin_warehouse_shipping_packing_list_id, - old_origin_warehouse_shipping_packing_list_id, - warehouse_order_number, + new_warehouse_shipping_packing_list_id, + old_warehouse_shipping_packing_list_id, booking_id, warehouse_sub_company_id, + warehouse_order_number, -- dimensions has_replaced_by_new_id, reference_contract, + packing_list_type, status, -- measures @@ -130,7 +168,8 @@ final__stg_shipping__origin_warehouse_shipping_packing_lists AS ( -- metadata _dbt_ran_datetime - FROM warehouse_shipping_list_replications_cover_warehouse_shipping_lists + FROM warehouse_shipping_list_cleaned + ) SELECT * FROM final__stg_shipping__origin_warehouse_shipping_packing_lists \ No newline at end of file