diff --git a/dbt_project.yml b/dbt_project.yml index fa4cd16..41345de 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -39,14 +39,28 @@ models: autocount: +materialized: view +schema: autocount__staging + +tags: + - autocount + + base: + +materialized: view + +schema: autocount__base crisp: +materialized: view +schema: crisp__staging + +tags: + - crisp + + base: + +materialized: view + +schema: crisp__base exchange: +materialized: view +schema: exchange__staging + +tags: + - exchange base: +materialized: view @@ -55,7 +69,9 @@ models: shipping: +materialized: view +schema: shipping__staging - + +tags: + - shipping + base: +materialized: view +schema: shipping__base diff --git a/models/intermediate/int_shipping__booking_delivery_and_billing_addresses.sql b/models/intermediate/int_shipping__booking_delivery_and_billing_addresses.sql new file mode 100644 index 0000000..ada8423 --- /dev/null +++ b/models/intermediate/int_shipping__booking_delivery_and_billing_addresses.sql @@ -0,0 +1,123 @@ +-- IMPORT +WITH booking_addresses AS ( + SELECT * FROM {{ ref('stg_shipping__booking_addresses') }} +), + + +-- LOGIC +booking_addresses_clean AS ( + SELECT + *, + ROW_NUMBER() OVER (PARTITION BY booking_id, address_type ORDER BY created_datetime DESC) AS row_number_index + + FROM booking_addresses + + WHERE + deleted_datetime IS NULL + AND + status = 'APPROVED' + + QUALIFY row_number_index = 1 +), + +group_by_booking_id AS ( + SELECT + booking_id, + MAX(IFF(address_type = 'DELIVERY', booking_address_id, NULL)) AS delivery_booking_address_id, + MAX(IFF(address_type = 'DELIVERY', district_id, NULL)) AS delivery_address_district_id, + MAX(IFF(address_type = 'DELIVERY', state_id, NULL)) AS delivery_address_state_id, + MAX(IFF(address_type = 'DELIVERY', country_id, NULL)) AS delivery_address_country_id, + MAX(IFF(address_type = 'DELIVERY', address_reference, NULL)) AS delivery_address_reference, + MAX(IFF(address_type = 'DELIVERY', address_line_one, NULL)) AS delivery_address_line_one, + MAX(IFF(address_type = 'DELIVERY', address_line_two, NULL)) AS delivery_address_line_two, + MAX(IFF(address_type = 'DELIVERY', district_name, NULL)) AS delivery_address_district_name, + MAX(IFF(address_type = 'DELIVERY', state_name, NULL)) AS delivery_address_state_name, + MAX(IFF(address_type = 'DELIVERY', country_name, NULL)) AS delivery_address_country_name, + MAX(IFF(address_type = 'DELIVERY', postcode, NULL)) AS delivery_address_postcode, + MAX(IFF(address_type = 'DELIVERY', address_type, NULL)) AS delivery_address_type, + MAX(IFF(address_type = 'DELIVERY', is_default_address, NULL)) AS is_default_delivery_address, + MAX(IFF(address_type = 'DELIVERY', status, NULL)) AS delivery_address_status, + MAX(IFF(address_type = 'DELIVERY', deleted_datetime, NULL)) AS delivery_address_deleted_datetime, + MAX(IFF(address_type = 'DELIVERY', created_datetime, NULL)) AS delivery_address_created_datetime, + MAX(IFF(address_type = 'DELIVERY', updated_datetime, NULL)) AS delivery_address_updated_datetime, + + MAX(IFF(address_type = 'BILLING', booking_address_id, NULL)) AS billing_booking_address_id, + MAX(IFF(address_type = 'BILLING', district_id, NULL)) AS billing_address_district_id, + MAX(IFF(address_type = 'BILLING', state_id, NULL)) AS billing_address_state_id, + MAX(IFF(address_type = 'BILLING', country_id, NULL)) AS billing_address_country_id, + MAX(IFF(address_type = 'BILLING', address_reference, NULL)) AS billing_address_reference, + MAX(IFF(address_type = 'BILLING', address_line_one, NULL)) AS billing_address_line_one, + MAX(IFF(address_type = 'BILLING', address_line_two, NULL)) AS billing_address_line_two, + MAX(IFF(address_type = 'BILLING', district_name, NULL)) AS billing_address_district_name, + MAX(IFF(address_type = 'BILLING', state_name, NULL)) AS billing_address_state_name, + MAX(IFF(address_type = 'BILLING', country_name, NULL)) AS billing_address_country_name, + MAX(IFF(address_type = 'BILLING', postcode, NULL)) AS billing_address_postcode, + MAX(IFF(address_type = 'BILLING', address_type, NULL)) AS billing_address_type, + MAX(IFF(address_type = 'BILLING', is_default_address, NULL)) AS is_default_billing_address, + MAX(IFF(address_type = 'BILLING', status, NULL)) AS billing_address_status, + MAX(IFF(address_type = 'BILLING', deleted_datetime, NULL)) AS billing_address_deleted_datetime, + MAX(IFF(address_type = 'BILLING', created_datetime, NULL)) AS billing_address_created_datetime, + MAX(IFF(address_type = 'BILLING', updated_datetime, NULL)) AS billing_address_updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM booking_addresses_clean + + GROUP BY + booking_id +), + + +-- FINAL +final__int_shipping__booking_delivery_and_billing_addresses AS ( + SELECT + -- ids + booking_id, + delivery_booking_address_id, + delivery_address_district_id, + delivery_address_state_id, + delivery_address_country_id, + billing_booking_address_id, + billing_address_district_id, + billing_address_state_id, + billing_address_country_id, + + -- dimensions + delivery_address_reference, + delivery_address_line_one, + delivery_address_line_two, + delivery_address_district_name, + delivery_address_state_name, + delivery_address_country_name, + delivery_address_postcode, + delivery_address_type, + is_default_delivery_address, + delivery_address_status, + billing_address_reference, + billing_address_line_one, + billing_address_line_two, + billing_address_district_name, + billing_address_state_name, + billing_address_country_name, + billing_address_postcode, + billing_address_type, + is_default_billing_address, + billing_address_status, + + -- measures + + -- date/times + delivery_address_deleted_datetime, + delivery_address_created_datetime, + delivery_address_updated_datetime, + billing_address_deleted_datetime, + billing_address_created_datetime, + billing_address_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM group_by_booking_id +) + +SELECT * FROM final__int_shipping__booking_delivery_and_billing_addresses \ No newline at end of file diff --git a/models/intermediate/int_shipping__containers_group_by_packing_list_ids.sql b/models/intermediate/int_shipping__containers_group_by_packing_list_ids.sql new file mode 100644 index 0000000..ee93c74 --- /dev/null +++ b/models/intermediate/int_shipping__containers_group_by_packing_list_ids.sql @@ -0,0 +1,114 @@ +-- IMPORT +WITH containers AS ( + SELECT * FROM {{ ref('stg_shipping__containers') }} +), + +containers_brg_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__containers_brg_packing_lists') }} +), + +container_transport_arrangments AS ( + SELECT * FROM {{ ref('stg_shipping__container_transport_arrangments') }} + -- WHERE deleted_datetime IS NULL +), + +-- LOGIC +container_get_packing_list_ids AS ( + SELECT + containers_brg_packing_lists.packing_list_id, + containers.* + + FROM containers + + LEFT JOIN containers_brg_packing_lists + ON (containers.container_id = containers_brg_packing_lists.container_id) +), + +contianer_get_transport_arrangments AS ( + SELECT + container_get_packing_list_ids.packing_list_id, + container_get_packing_list_ids.container_id, + container_get_packing_list_ids.sub_company_id AS container_warehouse_sub_company_id, + container_get_packing_list_ids.container_reference, + container_get_packing_list_ids.container_number, + container_get_packing_list_ids.container_type, + container_get_packing_list_ids.seal_reference AS container_seal_reference, + container_get_packing_list_ids.status AS container_status, + container_get_packing_list_ids.loaded_datetime AS container_loaded_datetime, + container_get_packing_list_ids.deleted_datetime AS container_deleted_datetime, + container_get_packing_list_ids.created_datetime AS container_created_datetime, + container_get_packing_list_ids.updated_datetime AS container_updated_datetime, + container_transport_arrangments.transport_arrangement_id AS container_transport_arrangement_id, + container_transport_arrangments.dispatched_datetime AS container_departed_datetime, + container_transport_arrangments.droped_datetime AS container_arrived_datetime, + container_transport_arrangments.courier AS container_deliver_courier, + container_transport_arrangments.tracking_number AS container_tracking_number, + container_transport_arrangments.status AS container_transport_arrangment_status, + container_transport_arrangments.number_of_etd_datetime_postpone AS number_of_container_etd_datetime_postpone, + container_transport_arrangments.number_of_eta_datetime_postpone AS number_of_container_eta_datetime_postpone, + container_transport_arrangments.etd_datetimes AS container_etd_datetimes, + container_transport_arrangments.first_etd_datetime AS first_container_etd_datetimes, + container_transport_arrangments.last_etd_datetime AS last_container_etd_datetimes, + container_transport_arrangments.eta_datetimes AS container_eta_datetimes, + container_transport_arrangments.first_eta_datetime AS first_container_eta_datetime, + container_transport_arrangments.last_eta_datetime AS last_container_eta_datetime, + container_transport_arrangments.first_estimate_schedule_created_datetime AS first_container_estimate_schedule_created_datetime, + container_transport_arrangments.last_estimate_schedule_created_datetime AS last_container_estimate_schedule_created_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM container_get_packing_list_ids + + LEFT JOIN container_transport_arrangments + ON (container_get_packing_list_ids.container_id = container_transport_arrangments.container_id) +), + +-- FINAL +final__int_shipping__containers_group_by_packing_list_ids AS ( + SELECT + -- ids + packing_list_id, + container_id, + container_warehouse_sub_company_id, + container_transport_arrangement_id, + + -- dimensions + container_reference, + container_number, + container_seal_reference, + container_type, + container_status, + container_transport_arrangment_status, + container_deliver_courier, + container_tracking_number, + + + -- measures + number_of_container_etd_datetime_postpone, + number_of_container_eta_datetime_postpone, + + -- date/times + container_loaded_datetime, + container_departed_datetime, + container_arrived_datetime, + container_etd_datetimes, + first_container_etd_datetimes, + last_container_etd_datetimes, + container_eta_datetimes, + first_container_eta_datetime, + last_container_eta_datetime, + first_container_estimate_schedule_created_datetime, + last_container_estimate_schedule_created_datetime, + container_deleted_datetime, + container_created_datetime, + container_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM contianer_get_transport_arrangments +) +-- -- YAM already do the hardcode clean, but not validate by omair +-- SELECT * FROM container_group_by_packing_list_id WHERE packing_list_id in (21276, 16087) OR packing_list_id IS NULL + +SELECT * FROM final__int_shipping__containers_group_by_packing_list_ids \ No newline at end of file diff --git a/models/intermediate/int_shipping__destination_warehouse_packing_list_transport_arrangements.sql b/models/intermediate/int_shipping__destination_warehouse_packing_list_transport_arrangements.sql new file mode 100644 index 0000000..3fef753 --- /dev/null +++ b/models/intermediate/int_shipping__destination_warehouse_packing_list_transport_arrangements.sql @@ -0,0 +1,90 @@ +-- IMPORT +WITH origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__origin_warehouse_shipping_packing_lists') }} +), + +packing_list_transport_arrangments AS ( + SELECT * FROM {{ ref('stg_shipping__packing_list_transport_arrangments') }} +), + + +-- LOGIC +remove_non_destination_warehouse_transport_arrangments AS ( + SELECT + origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id, + packing_list_transport_arrangments.transport_arrangement_id AS destination_warehouse_transport_arrangement_id, + packing_list_transport_arrangments.transport_mode AS destination_warehouse_transport_transport_mode, + packing_list_transport_arrangments.courier AS destination_warehouse_transport_courier, + packing_list_transport_arrangments.tracking_number AS destination_warehouse_transport_tracking_number, + packing_list_transport_arrangments.status AS destination_warehouse_transport_status, + packing_list_transport_arrangments.number_of_etd_datetime_postpone AS destination_warehouse_transport_number_of_etd_datetime_postpone, + packing_list_transport_arrangments.number_of_eta_datetime_postpone AS destination_warehouse_transport_number_of_eta_datetime_postpone, + packing_list_transport_arrangments.dispatched_datetime AS destination_warehouse_transport_dispatched_datetime, + packing_list_transport_arrangments.droped_datetime AS destination_warehouse_transport_droped_datetime, + packing_list_transport_arrangments.etd_datetimes AS destination_warehouse_transport_etd_datetimes, + packing_list_transport_arrangments.first_etd_datetime AS destination_warehouse_transport_first_etd_datetime, + packing_list_transport_arrangments.last_etd_datetime AS destination_warehouse_transport_last_etd_datetime, + packing_list_transport_arrangments.eta_datetimes AS destination_warehouse_transport_eta_datetimes, + packing_list_transport_arrangments.first_eta_datetime AS destination_warehouse_transport_first_eta_datetime, + packing_list_transport_arrangments.last_eta_datetime AS destination_warehouse_transport_last_eta_datetime, + packing_list_transport_arrangments.first_estimate_schedule_created_datetime AS destination_warehouse_transport_first_estimate_schedule_created_datetime, + packing_list_transport_arrangments.last_estimate_schedule_created_datetime AS destination_warehouse_transport_last_estimate_schedule_created_datetime, + packing_list_transport_arrangments.created_datetime AS destination_warehouse_transport_created_datetime, + packing_list_transport_arrangments.updated_datetime AS destination_warehouse_transport_updated_datetime + + FROM origin_warehouse_shipping_packing_lists + + INNER JOIN packing_list_transport_arrangments + ON (origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = packing_list_transport_arrangments.packing_list_id) +), + +get_lastest_created_row_only AS ( --Because the dataset not accurate, many duplicate withough deleted all those duplicate row + SELECT + *, + ROW_NUMBER() OVER (PARTITION BY old_origin_warehouse_shipping_packing_list_id ORDER BY destination_warehouse_transport_created_datetime DESC) AS row_number_index, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM remove_non_destination_warehouse_transport_arrangments + + QUALIFY row_number_index = 1 +), + +-- FINAL +final__int_shipping__destination_warehouse_packing_list_transport_arrangements AS ( + SELECT + -- ids + destination_warehouse_transport_arrangement_id, + old_origin_warehouse_shipping_packing_list_id, + + -- dimensions + destination_warehouse_transport_transport_mode, + destination_warehouse_transport_courier, + destination_warehouse_transport_tracking_number, + destination_warehouse_transport_status, + + -- measures + destination_warehouse_transport_number_of_etd_datetime_postpone, + destination_warehouse_transport_number_of_eta_datetime_postpone, + + -- date/times + destination_warehouse_transport_dispatched_datetime, + destination_warehouse_transport_droped_datetime, + destination_warehouse_transport_etd_datetimes, + destination_warehouse_transport_first_etd_datetime, + destination_warehouse_transport_last_etd_datetime, + destination_warehouse_transport_eta_datetimes, + destination_warehouse_transport_first_eta_datetime, + destination_warehouse_transport_last_eta_datetime, + destination_warehouse_transport_first_estimate_schedule_created_datetime, + destination_warehouse_transport_last_estimate_schedule_created_datetime, + destination_warehouse_transport_created_datetime, + destination_warehouse_transport_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM get_lastest_created_row_only +) + +SELECT * FROM final__int_shipping__destination_warehouse_packing_list_transport_arrangements \ No newline at end of file diff --git a/models/intermediate/int_shipping__origin_factory_packing_list_transport_arrangements.sql b/models/intermediate/int_shipping__origin_factory_packing_list_transport_arrangements.sql new file mode 100644 index 0000000..561879e --- /dev/null +++ b/models/intermediate/int_shipping__origin_factory_packing_list_transport_arrangements.sql @@ -0,0 +1,81 @@ +-- IMPORT +WITH origin_warehouse_receive_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__origin_warehouse_receive_packing_lists') }} +), + +packing_list_transport_arrangments AS ( + SELECT * FROM {{ ref('stg_shipping__packing_list_transport_arrangments') }} +), + + +-- LOGIC +remove_non_origin_factory_transport_arrangments AS ( + SELECT + origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id, + packing_list_transport_arrangments.transport_arrangement_id AS origin_factory_transport_arrangement_id, + packing_list_transport_arrangments.transport_mode AS origin_factory_transport_transport_mode, + packing_list_transport_arrangments.courier AS origin_factory_transport_courier, + packing_list_transport_arrangments.tracking_number AS origin_factory_transport_tracking_number, + packing_list_transport_arrangments.status AS origin_factory_transport_status, + packing_list_transport_arrangments.number_of_etd_datetime_postpone AS origin_factory_transport_number_of_etd_datetime_postpone, + packing_list_transport_arrangments.number_of_eta_datetime_postpone AS origin_factory_transport_number_of_eta_datetime_postpone, + packing_list_transport_arrangments.dispatched_datetime AS origin_factory_transport_dispatched_datetime, + packing_list_transport_arrangments.droped_datetime AS origin_factory_transport_droped_datetime, + packing_list_transport_arrangments.etd_datetimes AS origin_factory_transport_etd_datetimes, + packing_list_transport_arrangments.first_etd_datetime AS origin_factory_transport_first_etd_datetime, + packing_list_transport_arrangments.last_etd_datetime AS origin_factory_transport_last_etd_datetime, + packing_list_transport_arrangments.eta_datetimes AS origin_factory_transport_eta_datetimes, + packing_list_transport_arrangments.first_eta_datetime AS origin_factory_transport_first_eta_datetime, + packing_list_transport_arrangments.last_eta_datetime AS origin_factory_transport_last_eta_datetime, + packing_list_transport_arrangments.first_estimate_schedule_created_datetime AS origin_factory_transport_first_estimate_schedule_created_datetime, + packing_list_transport_arrangments.last_estimate_schedule_created_datetime AS origin_factory_transport_last_estimate_schedule_created_datetime, + packing_list_transport_arrangments.created_datetime AS origin_factory_transport_created_datetime, + packing_list_transport_arrangments.updated_datetime AS origin_factory_transport_updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM origin_warehouse_receive_packing_lists + + INNER JOIN packing_list_transport_arrangments + ON (origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id = packing_list_transport_arrangments.packing_list_id) +), + +-- FINAL +final__int_shipping__origin_factory_packing_list_transport_arrangements AS ( + SELECT + -- ids + origin_factory_transport_arrangement_id, + old_origin_warehouse_receive_packing_list_id, + + -- dimensions + origin_factory_transport_transport_mode, + origin_factory_transport_courier, + origin_factory_transport_tracking_number, + origin_factory_transport_status, + + -- measures + origin_factory_transport_number_of_etd_datetime_postpone, + origin_factory_transport_number_of_eta_datetime_postpone, + + -- date/times + origin_factory_transport_dispatched_datetime, + origin_factory_transport_droped_datetime, + origin_factory_transport_etd_datetimes, + origin_factory_transport_first_etd_datetime, + origin_factory_transport_last_etd_datetime, + origin_factory_transport_eta_datetimes, + origin_factory_transport_first_eta_datetime, + origin_factory_transport_last_eta_datetime, + origin_factory_transport_first_estimate_schedule_created_datetime, + origin_factory_transport_last_estimate_schedule_created_datetime, + + origin_factory_transport_created_datetime, + origin_factory_transport_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM remove_non_origin_factory_transport_arrangments +) + +SELECT * FROM final__int_shipping__origin_factory_packing_list_transport_arrangements \ No newline at end of file diff --git a/models/intermediate/int_shipping__packages_group_by_packing_list_ids.sql b/models/intermediate/int_shipping__packages_group_by_packing_list_ids.sql new file mode 100644 index 0000000..1dd0eff --- /dev/null +++ b/models/intermediate/int_shipping__packages_group_by_packing_list_ids.sql @@ -0,0 +1,129 @@ +-- IMPORT +WITH packages AS ( + SELECT * FROM {{ ref('stg_shipping__packages') }} +), + +package_extra_charges AS ( + SELECT * FROM {{ ref('stg_shipping__package_extra_charges') }} +), + +-- LOGIC +packages_group_by_packing_list_id AS ( + SELECT + packing_list_id, + COUNT(DISTINCT(package_id)) AS number_of_package_record, + ARRAY_AGG(DISTINCT(package_description)) AS package_descriptions, + SUM(width_cm) AS total_package_width_cm, + SUM(height_cm) AS total_package_height_cm, + SUM(length_cm) AS total_package_length_cm, + sum(weight_kg) AS total_package_weight_kg, + SUM(quantity) AS total_package_item, + SUM(cbm) AS total_package_cbm + + FROM + packages + + GROUP BY + packing_list_id +), + +package_extra_charges_group_by_packing_list_id AS ( + SELECT + packing_list_id, + COUNT(DISTINCT( + IFF(package_type = 'OVER_WEIGHT', + package_id, + null) + )) AS number_of_over_weight_charge_record, + + SUM( + IFF(package_type = 'OVER_WEIGHT', + extra_charge_cbm, + 0) + ) AS over_weight_cbm, + + SUM( + IFF(package_type = 'OVER_WEIGHT', + over_weight_kg, + 0) + ) AS over_weight_kg, + + + COUNT(DISTINCT( + IFF(package_type = 'PALLET', + package_id, + null) + )) AS number_of_pallet_charge_record, + + SUM( + IFF(package_type = 'PALLET', + extra_charge_cbm, + 0) + ) AS pallet_charge_cbm + + FROM package_extra_charges + + GROUP BY + packing_list_id +), + +pakacges_and_package_extra_charges_join_using_packing_list_id AS ( + SELECT + packages_group_by_packing_list_id.packing_list_id, + packages_group_by_packing_list_id.number_of_package_record, + packages_group_by_packing_list_id.package_descriptions, + packages_group_by_packing_list_id.total_package_width_cm, + packages_group_by_packing_list_id.total_package_height_cm, + packages_group_by_packing_list_id.total_package_length_cm, + packages_group_by_packing_list_id.total_package_weight_kg, + packages_group_by_packing_list_id.total_package_item, + packages_group_by_packing_list_id.total_package_cbm, + + package_extra_charges_group_by_packing_list_id.number_of_over_weight_charge_record, + package_extra_charges_group_by_packing_list_id.over_weight_cbm, + package_extra_charges_group_by_packing_list_id.over_weight_kg, + package_extra_charges_group_by_packing_list_id.number_of_pallet_charge_record, + package_extra_charges_group_by_packing_list_id.pallet_charge_cbm, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM packages_group_by_packing_list_id + + LEFT JOIN package_extra_charges_group_by_packing_list_id + ON (packages_group_by_packing_list_id.packing_list_id = package_extra_charges_group_by_packing_list_id.packing_list_id) +), + +-- FINAL +final__int_shipping__packages_group_by_packing_lists AS ( + SELECT + -- ids + packing_list_id, + + -- dimensions + package_descriptions, + + -- measures + number_of_package_record, + total_package_width_cm, + total_package_height_cm, + total_package_length_cm, + total_package_weight_kg, + total_package_item, + total_package_cbm, + + number_of_over_weight_charge_record, + over_weight_cbm, + over_weight_kg, + number_of_pallet_charge_record, + pallet_charge_cbm, + + -- date/times + + -- metadata + _dbt_ran_datetime + + + FROM pakacges_and_package_extra_charges_join_using_packing_list_id +) + +SELECT * FROM final__int_shipping__packages_group_by_packing_lists \ No newline at end of file diff --git a/models/intermediate/int_shipping__packing_list_invoice_payments_group_by_packing_list_ids.sql b/models/intermediate/int_shipping__packing_list_invoice_payments_group_by_packing_list_ids.sql new file mode 100644 index 0000000..4a7082c --- /dev/null +++ b/models/intermediate/int_shipping__packing_list_invoice_payments_group_by_packing_list_ids.sql @@ -0,0 +1,109 @@ +-- IMPORT +WITH stg_shipping__packing_list_invoice_payments AS ( + SELECT * FROM {{ ref('stg_shipping__packing_list_invoice_payments') }} +), + +-- LOGIC +remove_expired_invoice_payments AS ( + SELECT + * + + FROM stg_shipping__packing_list_invoice_payments + + WHERE + invoice_status != 'EXPIRED' + AND + payment_status NOT IN ('SUSPENDED','REJECTED') +), + +group_by_packing_list_ids AS ( + SELECT + old_origin_warehouse_shipping_packing_list_id, + invoice_sub_company_id, + payment_sub_company_id, + COUNT(DISTINCT(packing_list_invoice_id)) AS number_of_invoice, + COUNT(DISTINCT(packing_list_payment_id)) AS number_of_payment, + + SUM(invoice_number_of_shipping_fee) AS invoice_number_of_shipping_fee, + SUM(invoice_average_shipping_fee_price_per_quantity)/SUM(invoice_number_of_shipping_fee) AS invoice_average_shipping_fee_price_per_quantity, + SUM(invoice_total_shipping_fee_quantity) AS invoice_total_shipping_fee_quantity, + SUM(invoice_total_shipping_fee_value_rm) AS invoice_total_shipping_fee_value_rm, + SUM(invoice_number_of_min_cbm_charge) AS invoice_number_of_min_cbm_charge, + SUM(invoice_average_min_cbm_charge_price_per_quantity)/SUM(invoice_number_of_min_cbm_charge) AS invoice_average_min_cbm_charge_price_per_quantity, + SUM(invoice_total_min_cbm_charge_quantity) AS invoice_total_min_cbm_charge_quantity, + SUM(invoice_total_min_cbm_charge_value_rm) AS invoice_total_min_cbm_charge_value_rm, + SUM(invoice_number_of_over_weight_charge) AS invoice_number_of_over_weight_charge, + SUM(invoice_average_over_weight_charge_price_per_quantity)/SUM(invoice_number_of_over_weight_charge) AS invoice_average_over_weight_charge_price_per_quantity, + SUM(invoice_total_over_weight_charge_quantity) AS invoice_total_over_weight_charge_quantity, + SUM(invoice_total_over_weight_charge_value_rm) AS invoice_total_over_weight_charge_value_rm, + SUM(invoice_number_of_custom_charge) AS invoice_number_of_custom_charge, + SUM(invoice_average_custom_charge_price_per_quantity)/SUM(invoice_number_of_custom_charge) AS invoice_average_custom_charge_price_per_quantity, + SUM(invoice_total_custom_charge_quantity) AS invoice_total_custom_charge_quantity, + SUM(invoice_total_custom_charge_value_rm) AS invoice_total_custom_charge_value_rm, + SUM(invoice_value_rm) AS invoice_value_rm, + SUM(invoice_tax_rm) AS invoice_tax_rm, + SUM(invoice_service_charge) AS invoice_service_charge, + SUM(payment_value_rm) AS payment_value_rm, + SUM(payment_tax_rm) AS payment_tax_rm, + SUM(payment_service_charge_rm) AS payment_service_charge_rm, + MAX(invoice_created_datetime) AS last_invoice_created_datetime, + MAX(payment_created_datetime) AS last_payment_created_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + + FROM remove_expired_invoice_payments + + GROUP BY + old_origin_warehouse_shipping_packing_list_id, + invoice_sub_company_id, + payment_sub_company_id + +), + +-- FINAL +final__int_shipping__packing_list_invoice_payments_group_by_packing_list_ids AS ( + SELECT + -- ids + old_origin_warehouse_shipping_packing_list_id, + invoice_sub_company_id, + payment_sub_company_id, + + -- dimensions + + -- measures + number_of_invoice, + number_of_payment, + invoice_number_of_shipping_fee, + invoice_average_shipping_fee_price_per_quantity, + invoice_total_shipping_fee_quantity, + invoice_total_shipping_fee_value_rm, + invoice_number_of_min_cbm_charge, + invoice_average_min_cbm_charge_price_per_quantity, + invoice_total_min_cbm_charge_quantity, + invoice_total_min_cbm_charge_value_rm, + invoice_number_of_over_weight_charge, + invoice_average_over_weight_charge_price_per_quantity, + invoice_total_over_weight_charge_quantity, + invoice_total_over_weight_charge_value_rm, + invoice_number_of_custom_charge, + invoice_average_custom_charge_price_per_quantity, + invoice_total_custom_charge_quantity, + invoice_total_custom_charge_value_rm, + invoice_value_rm, + invoice_tax_rm, + invoice_service_charge, + payment_value_rm, + payment_tax_rm, + payment_service_charge_rm, + + -- date/times + last_invoice_created_datetime, + last_payment_created_datetime, + + -- metadata + _dbt_ran_datetime + + FROM group_by_packing_list_ids +) + +SELECT * FROM final__int_shipping__packing_list_invoice_payments_group_by_packing_list_ids \ No newline at end of file diff --git a/models/intermediate/int_shipping__vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.sql b/models/intermediate/int_shipping__vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.sql new file mode 100644 index 0000000..eddedf3 --- /dev/null +++ b/models/intermediate/int_shipping__vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.sql @@ -0,0 +1,128 @@ +-- IMPORT +WITH origin_warehouse_receive_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__origin_warehouse_receive_packing_lists') }} +), + +origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__origin_warehouse_shipping_packing_lists') }} +), + + +-- LOGIC +-- -- receive list +vt_receive_packing_lists AS ( + SELECT + origin_warehouse_receive_packing_lists.* + + FROM origin_warehouse_receive_packing_lists + + WHERE warehouse_sub_company_id IN (2, 3, 4) -- 3,4 is vt warehouse sub_company_id +), + +count_receive_list_booking_id_row AS ( + SELECT + vt_receive_packing_lists.booking_id, + count(vt_receive_packing_lists.booking_id) AS number_of_booking_id_occur + + FROM vt_receive_packing_lists + GROUP BY booking_id +), + +vt_receive_packing_list_remove_booking_id_more_than_one AS ( + SELECT + vt_receive_packing_lists.origin_warehouse_receive_packing_list_id, + vt_receive_packing_lists.old_origin_warehouse_receive_packing_list_id, + vt_receive_packing_lists.booking_id, + vt_receive_packing_lists.warehouse_order_number, + count_receive_list_booking_id_row.number_of_booking_id_occur + + FROM vt_receive_packing_lists + + LEFT JOIN count_receive_list_booking_id_row + ON (vt_receive_packing_lists.booking_id = count_receive_list_booking_id_row.booking_id) + + HAVING number_of_booking_id_occur = 1 +), + +-- -- shipping_list +vt_shipping_packing_lists AS ( + SELECT + origin_warehouse_shipping_packing_lists.* + + FROM origin_warehouse_shipping_packing_lists + + WHERE warehouse_sub_company_id = 2 -- 2 is vt sub_company_id +), + +count_shipping_list_booking_id_row AS ( + SELECT + vt_shipping_packing_lists.booking_id, + count(vt_shipping_packing_lists.booking_id) AS number_of_booking_id_occur + + FROM vt_shipping_packing_lists + GROUP BY booking_id +), + +vt_shipping_packing_list_remove_booking_id_more_than_one AS ( + SELECT + vt_shipping_packing_lists.origin_warehouse_shipping_packing_list_id, + vt_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id, + vt_shipping_packing_lists.booking_id, + vt_shipping_packing_lists.warehouse_order_number, + count_shipping_list_booking_id_row.number_of_booking_id_occur + + FROM vt_shipping_packing_lists + + LEFT JOIN count_shipping_list_booking_id_row + ON (vt_shipping_packing_lists.booking_id = count_shipping_list_booking_id_row.booking_id) + + HAVING number_of_booking_id_occur = 1 +), + +-- -- create_new_mapping_table +receive_and_shipping_packing_list_mapping AS ( + SELECT + vt_receive_packing_list_remove_booking_id_more_than_one.origin_warehouse_receive_packing_list_id, + vt_receive_packing_list_remove_booking_id_more_than_one.old_origin_warehouse_receive_packing_list_id, + vt_receive_packing_list_remove_booking_id_more_than_one.booking_id, + + CONCAT( vt_receive_packing_list_remove_booking_id_more_than_one.warehouse_order_number, + '/', + vt_shipping_packing_list_remove_booking_id_more_than_one.warehouse_order_number) AS warehouse_order_number, + + vt_shipping_packing_list_remove_booking_id_more_than_one.origin_warehouse_shipping_packing_list_id, + vt_shipping_packing_list_remove_booking_id_more_than_one.old_origin_warehouse_shipping_packing_list_id, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM vt_receive_packing_list_remove_booking_id_more_than_one + + INNER JOIN vt_shipping_packing_list_remove_booking_id_more_than_one + ON (vt_receive_packing_list_remove_booking_id_more_than_one.booking_id = vt_shipping_packing_list_remove_booking_id_more_than_one.booking_id) +), + + +-- FINAL +final__int_shipping__vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists AS ( + SELECT + -- ids + origin_warehouse_receive_packing_list_id, + old_origin_warehouse_receive_packing_list_id, + booking_id, + origin_warehouse_shipping_packing_list_id, + old_origin_warehouse_shipping_packing_list_id, + + -- dimensions + warehouse_order_number, + + -- measures + + -- date/times + + -- metadata + _dbt_ran_datetime + + FROM receive_and_shipping_packing_list_mapping +) + +SELECT * FROM final__int_shipping__vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists \ No newline at end of file diff --git a/models/intermediate/int_shpping__latest_sub_company_billing_addresses.sql b/models/intermediate/int_shpping__latest_sub_company_billing_addresses.sql new file mode 100644 index 0000000..6bd388e --- /dev/null +++ b/models/intermediate/int_shpping__latest_sub_company_billing_addresses.sql @@ -0,0 +1,58 @@ +-- IMPORT +WITH stg_shipping__sub_company_addresses AS ( + SELECT * FROM {{ ref('stg_shipping__sub_company_addresses') }} +), + + +-- LOGIC +latest_billing_addresses AS ( + SELECT + *, + row_number() OVER (PARTITION BY sub_company_id ORDER BY created_datetime DESC) AS latest_row_number + + FROM stg_shipping__sub_company_addresses + + WHERE + address_type = 'BILLING' + AND + deleted_datetime IS NULL + + QUALIFY + latest_row_number = 1 +), + +-- FINAL +final__int_shpping__latest_sub_company_billing_addresses AS ( + SELECT + -- ids + sub_company_address_id, + sub_company_id, + district_id, + state_id, + country_id, + + -- dimensions + address_reference, + address_line_one, + address_line_two, + district_name, + state_name, + country_name, + postcode, + address_type, + is_default_address, + status, + + -- measures + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM latest_billing_addresses + +) + +SELECT * FROM final__int_shpping__latest_sub_company_billing_addresses \ No newline at end of file diff --git a/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql b/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql new file mode 100644 index 0000000..5285d51 --- /dev/null +++ b/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql @@ -0,0 +1,201 @@ +--IMPORT +WITH origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('fct_shipping__origin_warehouse_shipping_packing_lists') }} +), + +companies AS ( + SELECT * FROM {{ ref('dim_shipping__companies') }} +), + + +--LOGIC +join_tables AS ( + SELECT + origin_warehouse_shipping_packing_lists.*, + companies.sub_company_marking_id, + companies.parent_company_id, + companies.company_type, + companies.state_name AS sub_company_state_name + + FROM (SELECT * EXCLUDE _dbt_ran_datetime FROM origin_warehouse_shipping_packing_lists) AS origin_warehouse_shipping_packing_lists + + LEFT JOIN companies + ON (origin_warehouse_shipping_packing_lists.sub_company_id = companies.sub_company_id) +), + +create_metrics AS ( + SELECT + *, + {% for column_name in ["parent_company", "sub_company"] %} + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY join_tables.{{column_name}}_id ORDER BY join_tables.origin_warehouse_shipping_order_created_datetime) = 1 THEN 1 + ELSE 0 + END AS is_first_time_{{column_name}}, + + CASE + WHEN join_tables.origin_warehouse_shipping_order_status = 'COMPLETED' + AND + MIN(IFF(join_tables.origin_warehouse_shipping_order_status = 'COMPLETED', join_tables.origin_warehouse_shipping_order_created_datetime, NULL)) + OVER (PARTITION BY join_tables.{{column_name}}_id) + = join_tables.origin_warehouse_shipping_order_created_datetime + THEN 1 + ELSE 0 + END AS is_first_time_{{column_name}}_completed, + {% endfor %} + + -- temp debug, need make sure it do not have null in fct, cause cant add + (ZEROIFNULL(total_package_cbm) + ZEROIFNULL(over_weight_cbm) + ZEROIFNULL(pallet_charge_cbm)) AS total_order_cbm, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM join_tables +), + +--FINAL +final__rep_shipping__origin_warehouse_shipping_orders AS ( + SELECT + -- ids + origin_warehouse_shipping_packing_list_id, + old_origin_warehouse_shipping_packing_list_id, + booking_id, + parent_company_id, + sub_company_id, + sub_company_marking_id, + warehouse_sub_company_id, + invoice_sub_company_id, + payment_sub_company_id, + container_id, + container_warehouse_sub_company_id, + container_transport_arrangement_id, + destination_warehouse_transport_arrangement_id, + + -- dimensions + warehouse_order_number, + reference_contract, + has_replaced_by_new_origin_warehouse_shipping_packing_list_id, + company_type, + is_first_time_parent_company, + is_first_time_parent_company_completed, + is_first_time_sub_company, + is_first_time_sub_company_completed, + sub_company_state_name, + service_type, + package_descriptions, + delivery_address_reference, + delivery_address_line_one, + delivery_address_line_two, + delivery_address_district_name, + delivery_address_state_name, + delivery_address_country_name, + delivery_address_postcode, + delivery_address_type, + is_default_delivery_address, + delivery_address_status, + billing_address_reference, + billing_address_line_one, + billing_address_line_two, + billing_address_district_name, + billing_address_state_name, + billing_address_country_name, + billing_address_postcode, + billing_address_type, + is_default_billing_address, + container_reference, + container_number, + container_seal_reference, + container_type, + container_deliver_courier, + container_tracking_number, + destination_warehouse_transport_transport_mode, + destination_warehouse_transport_courier, + destination_warehouse_transport_tracking_number, + booking_status, + billing_address_status, + origin_warehouse_shipping_order_status, + container_status, + container_transport_arrangment_status, + destination_warehouse_transport_status, + + -- measures + number_of_package_record, + total_package_width_cm, + total_package_height_cm, + total_package_length_cm, + total_package_weight_kg, + total_package_item, + total_package_cbm, + number_of_over_weight_charge_record, + over_weight_cbm, + over_weight_kg, + number_of_pallet_charge_record, + pallet_charge_cbm, + total_order_cbm, + number_of_invoice, + number_of_payment, + invoice_number_of_shipping_fee, + invoice_average_shipping_fee_price_per_quantity, + invoice_total_shipping_fee_quantity, + invoice_total_shipping_fee_value_rm, + invoice_number_of_min_cbm_charge, + invoice_average_min_cbm_charge_price_per_quantity, + invoice_total_min_cbm_charge_quantity, + invoice_total_min_cbm_charge_value_rm, + invoice_number_of_over_weight_charge, + invoice_average_over_weight_charge_price_per_quantity, + invoice_total_over_weight_charge_quantity, + invoice_total_over_weight_charge_value_rm, + invoice_number_of_custom_charge, + invoice_average_custom_charge_price_per_quantity, + invoice_total_custom_charge_quantity, + invoice_total_custom_charge_value_rm, + invoice_value_rm, + invoice_tax_rm, + invoice_service_charge, + payment_value_rm, + payment_tax_rm, + payment_service_charge_rm, + number_of_container_etd_datetime_postpone, + number_of_container_eta_datetime_postpone, + destination_warehouse_transport_number_of_etd_datetime_postpone, + destination_warehouse_transport_number_of_eta_datetime_postpone, + + -- date/times + booking_created_datetime, + delivery_address_created_datetime, + billing_address_created_datetime, + origin_warehouse_shipping_order_created_datetime, + last_invoice_created_datetime, + last_payment_created_datetime, + container_loaded_datetime, + container_departed_datetime, + container_arrived_datetime, + container_etd_datetimes, + first_container_etd_datetimes, + last_container_etd_datetimes, + container_eta_datetimes, + first_container_eta_datetime, + last_container_eta_datetime, + first_container_estimate_schedule_created_datetime, + last_container_estimate_schedule_created_datetime, + container_deleted_datetime, + container_created_datetime, + container_updated_datetime, + destination_warehouse_transport_dispatched_datetime, + destination_warehouse_transport_droped_datetime, + destination_warehouse_transport_etd_datetimes, + destination_warehouse_transport_first_etd_datetime, + destination_warehouse_transport_last_etd_datetime, + destination_warehouse_transport_eta_datetimes, + destination_warehouse_transport_first_eta_datetime, + destination_warehouse_transport_last_eta_datetime, + destination_warehouse_transport_first_estimate_schedule_created_datetime, + destination_warehouse_transport_last_estimate_schedule_created_datetime, + destination_warehouse_transport_created_datetime, + + -- metadata + _dbt_ran_datetime + + FROM create_metrics +) + +SELECT * FROM final__rep_shipping__origin_warehouse_shipping_orders \ No newline at end of file diff --git a/models/marts/warehouse/dim_shipping__companies.sql b/models/marts/warehouse/dim_shipping__companies.sql new file mode 100644 index 0000000..f02197a --- /dev/null +++ b/models/marts/warehouse/dim_shipping__companies.sql @@ -0,0 +1,145 @@ +-- IMPORT +WITH parent_and_sub_companies AS ( + SELECT * FROM {{ ref('stg_shipping__parent_and_sub_companies') }} +), + +latest_sub_company_billing_addresses AS ( + SELECT * FROM {{ ref('int_shpping__latest_sub_company_billing_addresses') }} +), + +company_documents AS ( + SELECT * FROM {{ ref('stg_shipping__company_documents') }} +), + + +-- LOGIC +latest_company_documents AS ( + SELECT + document_id AS identity_document_id, + parent_company_id AS identity_parent_company_id, + approver_user_id AS identity_approver_user_id, + status AS identity_status, + document_type AS identity_document_type, + identity_number AS identity_identity_number, + approved_datetime AS identity_approved_datetime, + deleted_datetime AS identity_deleted_datetime, + created_datetime AS identity_created_datetime, + updated_datetime AS identity_updated_datetime, + row_number() OVER + (PARTITION BY parent_company_id + ORDER BY created_datetime DESC) AS latest_row_number + + FROM company_documents + + WHERE + status = 'APPROVED' + + QUALIFY + latest_row_number = 1 +), + +sub_companies_enrich AS ( + SELECT + parent_and_sub_companies.parent_company_id, + parent_and_sub_companies.autocount_id, + parent_and_sub_companies.sub_company_id, + parent_and_sub_companies.sub_company_marking_id, + parent_and_sub_companies.status AS sub_company_status, + parent_and_sub_companies.parent_company_name, + parent_and_sub_companies.sub_company_name, + parent_and_sub_companies.company_type, + parent_and_sub_companies.business_type, + parent_and_sub_companies.parent_company_created_datetime, + parent_and_sub_companies.parent_company_updated_datetime, + parent_and_sub_companies.sub_company_created_datetime, + parent_and_sub_companies.sub_company_updated_datetime, + + latest_sub_company_billing_addresses.sub_company_address_id, + latest_sub_company_billing_addresses.address_reference, + latest_sub_company_billing_addresses.address_line_one, + latest_sub_company_billing_addresses.address_line_two, + latest_sub_company_billing_addresses.district_name, + latest_sub_company_billing_addresses.state_name, + latest_sub_company_billing_addresses.country_name, + latest_sub_company_billing_addresses.postcode, + latest_sub_company_billing_addresses.address_type, + latest_sub_company_billing_addresses.is_default_address, + latest_sub_company_billing_addresses.status AS company_address_status, + latest_sub_company_billing_addresses.created_datetime AS company_addresses_created_datetime, + latest_sub_company_billing_addresses.updated_datetime AS company_addresses_updated_datetime, + + latest_company_documents.identity_document_id, + latest_company_documents.identity_approver_user_id, + latest_company_documents.identity_status, + latest_company_documents.identity_document_type, + latest_company_documents.identity_identity_number, + latest_company_documents.identity_approved_datetime, + latest_company_documents.identity_deleted_datetime, + latest_company_documents.identity_created_datetime, + latest_company_documents.identity_updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM parent_and_sub_companies + + LEFT JOIN latest_sub_company_billing_addresses + ON (parent_and_sub_companies.sub_company_id = latest_sub_company_billing_addresses.sub_company_id) + + LEFT JOIN latest_company_documents + ON (parent_and_sub_companies.parent_company_id = latest_company_documents.identity_parent_company_id) +), + +-- FINAL +final__dim_shipping__companies AS ( + SELECT + -- ids + sub_company_id, + sub_company_marking_id, + parent_company_id, + autocount_id, + sub_company_address_id, + identity_document_id, + identity_approver_user_id, + + -- dimensions + sub_company_status, + parent_company_name, + sub_company_name, + company_type, + business_type, + company_address_status, + address_reference, + address_line_one, + address_line_two, + district_name, + state_name, + country_name, + postcode, + address_type, + is_default_address, + identity_status, + identity_document_type, + identity_identity_number, + + -- measures + + -- date/times + parent_company_created_datetime, + parent_company_updated_datetime, + sub_company_created_datetime, + sub_company_updated_datetime, + company_addresses_created_datetime, + company_addresses_updated_datetime, + identity_approved_datetime, + identity_deleted_datetime, + identity_created_datetime, + identity_updated_datetime, + + -- metadata + _dbt_ran_datetime + + + FROM sub_companies_enrich +) + +SELECT * FROM final__dim_shipping__companies \ No newline at end of file diff --git a/models/marts/warehouse/fct_shipping__origin_warehouse_receive_packing_lists.sql b/models/marts/warehouse/fct_shipping__origin_warehouse_receive_packing_lists.sql new file mode 100644 index 0000000..891b001 --- /dev/null +++ b/models/marts/warehouse/fct_shipping__origin_warehouse_receive_packing_lists.sql @@ -0,0 +1,590 @@ +-- IMPORT +WITH origin_warehouse_receive_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__origin_warehouse_receive_packing_lists') }} +), + +origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__origin_warehouse_shipping_packing_lists') }} +), + +vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists AS ( + SELECT * FROM {{ ref('int_shipping__vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists') }} +), + +bookings AS ( + SELECT * FROM {{ ref('stg_shipping__bookings') }} +), + +booking_delivery_and_billing_addresses AS ( + SELECT * FROM {{ ref('int_shipping__booking_delivery_and_billing_addresses') }} +), + +origin_factory_packing_list_transport_arrangements AS ( + SELECT * FROM {{ ref('int_shipping__origin_factory_packing_list_transport_arrangements') }} +), + +destination_warehouse_packing_list_transport_arrangements AS ( + SELECT * FROM {{ ref('int_shipping__destination_warehouse_packing_list_transport_arrangements') }} +), + +packing_list_invoice_payments_group_by_packing_list_ids AS ( + SELECT * FROM {{ ref('int_shipping__packing_list_invoice_payments_group_by_packing_list_ids') }} +), + +packages_group_by_packing_list_ids AS ( + SELECT * FROM {{ ref('int_shipping__packages_group_by_packing_list_ids') }} +), + +container_group_by_packing_list_ids AS ( + SELECT * FROM {{ ref('int_shipping__containers_group_by_packing_list_ids') }} +), + + +-- LOGIC +origin_warehouse_receive_packing_lists_enrich AS ( + SELECT + origin_warehouse_receive_packing_lists.origin_warehouse_receive_packing_list_id, + origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id, + origin_warehouse_receive_packing_lists.has_replaced_by_new_id AS has_replaced_by_new_origin_warehouse_receive_packing_list_id, + + + IFF(vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.warehouse_order_number is not null, + vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.warehouse_order_number, + origin_warehouse_receive_packing_lists.warehouse_order_number) AS warehouse_order_number, + + origin_warehouse_receive_packing_lists.booking_id, + origin_warehouse_receive_packing_lists.warehouse_sub_company_id, + origin_warehouse_receive_packing_lists.reference_contract, + origin_warehouse_receive_packing_lists.status AS origin_warehouse_receive_order_status, + origin_warehouse_receive_packing_lists.created_datetime AS origin_warehouse_receive_order_created_datetime, + + bookings.sub_company_id, + bookings.service_type, + bookings.status AS booking_status, + bookings.created_datetime AS booking_created_datetime, + + booking_delivery_and_billing_addresses.delivery_address_reference, + booking_delivery_and_billing_addresses.delivery_address_line_one, + booking_delivery_and_billing_addresses.delivery_address_line_two, + booking_delivery_and_billing_addresses.delivery_address_district_name, + booking_delivery_and_billing_addresses.delivery_address_state_name, + booking_delivery_and_billing_addresses.delivery_address_country_name, + booking_delivery_and_billing_addresses.delivery_address_postcode, + booking_delivery_and_billing_addresses.delivery_address_type, + booking_delivery_and_billing_addresses.is_default_delivery_address, + booking_delivery_and_billing_addresses.delivery_address_status, + booking_delivery_and_billing_addresses.billing_address_reference, + booking_delivery_and_billing_addresses.billing_address_line_one, + booking_delivery_and_billing_addresses.billing_address_line_two, + booking_delivery_and_billing_addresses.billing_address_district_name, + booking_delivery_and_billing_addresses.billing_address_state_name, + booking_delivery_and_billing_addresses.billing_address_country_name, + booking_delivery_and_billing_addresses.billing_address_postcode, + booking_delivery_and_billing_addresses.billing_address_type, + booking_delivery_and_billing_addresses.is_default_billing_address, + booking_delivery_and_billing_addresses.billing_address_status, + booking_delivery_and_billing_addresses.delivery_address_created_datetime, + booking_delivery_and_billing_addresses.billing_address_created_datetime, + + packages_group_by_packing_list_ids.package_descriptions, + packages_group_by_packing_list_ids.number_of_package_record, + packages_group_by_packing_list_ids.total_package_width_cm, + packages_group_by_packing_list_ids.total_package_height_cm, + packages_group_by_packing_list_ids.total_package_length_cm, + packages_group_by_packing_list_ids.total_package_weight_kg, + packages_group_by_packing_list_ids.total_package_item, + packages_group_by_packing_list_ids.total_package_cbm, + packages_group_by_packing_list_ids.number_of_over_weight_charge_record, + packages_group_by_packing_list_ids.over_weight_cbm, + packages_group_by_packing_list_ids.over_weight_kg, + packages_group_by_packing_list_ids.number_of_pallet_charge_record, + packages_group_by_packing_list_ids.pallet_charge_cbm, + + origin_factory_packing_list_transport_arrangements.origin_factory_transport_arrangement_id, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_transport_mode, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_courier, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_tracking_number, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_status, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_number_of_etd_datetime_postpone, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_number_of_eta_datetime_postpone, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_dispatched_datetime, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_droped_datetime, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_etd_datetimes, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_first_etd_datetime, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_last_etd_datetime, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_eta_datetimes, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_first_eta_datetime, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_last_eta_datetime, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_first_estimate_schedule_created_datetime, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_last_estimate_schedule_created_datetime, + origin_factory_packing_list_transport_arrangements.origin_factory_transport_created_datetime + + FROM origin_warehouse_receive_packing_lists + + LEFT JOIN vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists + ON (origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id = vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.old_origin_warehouse_receive_packing_list_id) + + LEFT JOIN bookings + ON (origin_warehouse_receive_packing_lists.booking_id = bookings.booking_id) + + LEFT JOIN booking_delivery_and_billing_addresses + ON (bookings.booking_id = booking_delivery_and_billing_addresses.booking_id) + + LEFT JOIN packages_group_by_packing_list_ids + ON (origin_warehouse_receive_packing_lists.origin_warehouse_receive_packing_list_id = packages_group_by_packing_list_ids.packing_list_id) + + LEFT JOIN origin_factory_packing_list_transport_arrangements + ON (origin_warehouse_receive_packing_lists.old_origin_warehouse_receive_packing_list_id = origin_factory_packing_list_transport_arrangements.old_origin_warehouse_receive_packing_list_id) +), + +origin_warehouse_shipping_packing_lists_enrich AS ( + SELECT + origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id, + origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id, + origin_warehouse_shipping_packing_lists.has_replaced_by_new_id AS has_replaced_by_new_origin_warehouse_shipping_packing_list_id, + + IFF(vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.warehouse_order_number is not null, + vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.warehouse_order_number, + origin_warehouse_shipping_packing_lists.warehouse_order_number) AS warehouse_order_number, + + origin_warehouse_shipping_packing_lists.warehouse_sub_company_id, + origin_warehouse_shipping_packing_lists.reference_contract, + origin_warehouse_shipping_packing_lists.status AS origin_warehouse_shipping_order_status, + origin_warehouse_shipping_packing_lists.created_datetime AS origin_warehouse_shipping_order_created_datetime, + + packing_list_invoice_payments_group_by_packing_list_ids.invoice_sub_company_id, + packing_list_invoice_payments_group_by_packing_list_ids.payment_sub_company_id, + packing_list_invoice_payments_group_by_packing_list_ids.number_of_invoice, + packing_list_invoice_payments_group_by_packing_list_ids.number_of_payment, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_number_of_shipping_fee, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_average_shipping_fee_price_per_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_shipping_fee_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_shipping_fee_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_number_of_min_cbm_charge, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_average_min_cbm_charge_price_per_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_min_cbm_charge_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_min_cbm_charge_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_number_of_over_weight_charge, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_average_over_weight_charge_price_per_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_over_weight_charge_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_over_weight_charge_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_number_of_custom_charge, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_average_custom_charge_price_per_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_custom_charge_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_custom_charge_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_tax_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_service_charge, + packing_list_invoice_payments_group_by_packing_list_ids.payment_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.payment_tax_rm, + packing_list_invoice_payments_group_by_packing_list_ids.payment_service_charge_rm, + packing_list_invoice_payments_group_by_packing_list_ids.last_invoice_created_datetime, + packing_list_invoice_payments_group_by_packing_list_ids.last_payment_created_datetime, + + packages_group_by_packing_list_ids.package_descriptions, + packages_group_by_packing_list_ids.number_of_package_record, + packages_group_by_packing_list_ids.total_package_width_cm, + packages_group_by_packing_list_ids.total_package_height_cm, + packages_group_by_packing_list_ids.total_package_length_cm, + packages_group_by_packing_list_ids.total_package_weight_kg, + packages_group_by_packing_list_ids.total_package_item, + packages_group_by_packing_list_ids.total_package_cbm, + packages_group_by_packing_list_ids.number_of_over_weight_charge_record, + packages_group_by_packing_list_ids.over_weight_cbm, + packages_group_by_packing_list_ids.over_weight_kg, + packages_group_by_packing_list_ids.number_of_pallet_charge_record, + packages_group_by_packing_list_ids.pallet_charge_cbm, + + container_group_by_packing_list_ids.container_id, + container_group_by_packing_list_ids.container_warehouse_sub_company_id, + container_group_by_packing_list_ids.container_transport_arrangement_id, + container_group_by_packing_list_ids.container_reference, + container_group_by_packing_list_ids.container_number, + container_group_by_packing_list_ids.container_seal_reference, + container_group_by_packing_list_ids.container_type, + container_group_by_packing_list_ids.container_status, + container_group_by_packing_list_ids.container_transport_arrangment_status, + container_group_by_packing_list_ids.container_deliver_courier, + container_group_by_packing_list_ids.container_tracking_number, + container_group_by_packing_list_ids.number_of_container_etd_datetime_postpone, + container_group_by_packing_list_ids.number_of_container_eta_datetime_postpone, + container_group_by_packing_list_ids.container_loaded_datetime, + container_group_by_packing_list_ids.container_departed_datetime, + container_group_by_packing_list_ids.container_arrived_datetime, + container_group_by_packing_list_ids.container_etd_datetimes, + container_group_by_packing_list_ids.first_container_etd_datetimes, + container_group_by_packing_list_ids.last_container_etd_datetimes, + container_group_by_packing_list_ids.container_eta_datetimes, + container_group_by_packing_list_ids.first_container_eta_datetime, + container_group_by_packing_list_ids.last_container_eta_datetime, + container_group_by_packing_list_ids.first_container_estimate_schedule_created_datetime, + container_group_by_packing_list_ids.last_container_estimate_schedule_created_datetime, + container_group_by_packing_list_ids.container_deleted_datetime, + container_group_by_packing_list_ids.container_created_datetime, + container_group_by_packing_list_ids.container_updated_datetime, + + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_arrangement_id, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_transport_mode, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_courier, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_tracking_number, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_status, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_number_of_etd_datetime_postpone, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_number_of_eta_datetime_postpone, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_dispatched_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_droped_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_etd_datetimes, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_first_etd_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_last_etd_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_eta_datetimes, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_first_eta_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_last_eta_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_first_estimate_schedule_created_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_last_estimate_schedule_created_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_created_datetime + + + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists + ON (origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = vt_origin_warehouse_receive_packing_list_brg_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id) + + LEFT JOIN packing_list_invoice_payments_group_by_packing_list_ids + ON (origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = packing_list_invoice_payments_group_by_packing_list_ids.old_origin_warehouse_shipping_packing_list_id) + + LEFT JOIN packages_group_by_packing_list_ids + ON (origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id = packages_group_by_packing_list_ids.packing_list_id) + + LEFT JOIN container_group_by_packing_list_ids + ON (origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = container_group_by_packing_list_ids.packing_list_id) + + LEFT JOIN destination_warehouse_packing_list_transport_arrangements + ON (origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = destination_warehouse_packing_list_transport_arrangements.old_origin_warehouse_shipping_packing_list_id) +), + + +receive_join_shipping_list AS ( + SELECT + origin_warehouse_receive_packing_lists_enrich.origin_warehouse_receive_packing_list_id AS adjusted_origin_warehouse_receive_id, + origin_warehouse_receive_packing_lists_enrich.old_origin_warehouse_receive_packing_list_id AS origin_warehouse_receive_id, + IFF(origin_warehouse_receive_packing_lists_enrich.has_replaced_by_new_origin_warehouse_receive_packing_list_id = 1 + OR + origin_warehouse_shipping_packing_lists_enrich.has_replaced_by_new_origin_warehouse_shipping_packing_list_id = 1, + 1, + 0) AS has_adjusted_measurment, + origin_warehouse_receive_packing_lists_enrich.warehouse_order_number, + origin_warehouse_receive_packing_lists_enrich.booking_id, + origin_warehouse_receive_packing_lists_enrich.warehouse_sub_company_id AS warehouse_location_sub_company_id, + origin_warehouse_receive_packing_lists_enrich.origin_warehouse_receive_order_status AS origin_warehouse_receive_status, + origin_warehouse_receive_packing_lists_enrich.origin_warehouse_receive_order_created_datetime AS origin_warehouse_receive_created_datetime, + origin_warehouse_receive_packing_lists_enrich.sub_company_id, + origin_warehouse_receive_packing_lists_enrich.service_type, + origin_warehouse_receive_packing_lists_enrich.booking_status, + origin_warehouse_receive_packing_lists_enrich.booking_created_datetime, + origin_warehouse_receive_packing_lists_enrich.delivery_address_reference, + origin_warehouse_receive_packing_lists_enrich.delivery_address_line_one, + origin_warehouse_receive_packing_lists_enrich.delivery_address_line_two, + origin_warehouse_receive_packing_lists_enrich.delivery_address_district_name, + origin_warehouse_receive_packing_lists_enrich.delivery_address_state_name, + origin_warehouse_receive_packing_lists_enrich.delivery_address_country_name, + origin_warehouse_receive_packing_lists_enrich.delivery_address_postcode, + origin_warehouse_receive_packing_lists_enrich.delivery_address_type, + origin_warehouse_receive_packing_lists_enrich.is_default_delivery_address, + origin_warehouse_receive_packing_lists_enrich.delivery_address_status, + origin_warehouse_receive_packing_lists_enrich.billing_address_reference, + origin_warehouse_receive_packing_lists_enrich.billing_address_line_one, + origin_warehouse_receive_packing_lists_enrich.billing_address_line_two, + origin_warehouse_receive_packing_lists_enrich.billing_address_district_name, + origin_warehouse_receive_packing_lists_enrich.billing_address_state_name, + origin_warehouse_receive_packing_lists_enrich.billing_address_country_name, + origin_warehouse_receive_packing_lists_enrich.billing_address_postcode, + origin_warehouse_receive_packing_lists_enrich.billing_address_type, + origin_warehouse_receive_packing_lists_enrich.is_default_billing_address, + origin_warehouse_receive_packing_lists_enrich.billing_address_status, + origin_warehouse_receive_packing_lists_enrich.delivery_address_created_datetime, + origin_warehouse_receive_packing_lists_enrich.billing_address_created_datetime, + CASE + WHEN origin_warehouse_receive_packing_lists_enrich.package_descriptions IS NOT NULL AND origin_warehouse_shipping_packing_lists_enrich.package_descriptions IS NOT NULL + THEN ARRAY_CAT(origin_warehouse_receive_packing_lists_enrich.package_descriptions, origin_warehouse_shipping_packing_lists_enrich.package_descriptions) + WHEN origin_warehouse_receive_packing_lists_enrich.package_descriptions IS NOT NULL + THEN origin_warehouse_receive_packing_lists_enrich.package_descriptions + ELSE origin_warehouse_shipping_packing_lists_enrich.package_descriptions + END AS package_descriptions, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.number_of_package_record],[origin_warehouse_receive_packing_lists_enrich.number_of_package_record])[0]::INT AS number_of_package_record, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.total_package_width_cm],[origin_warehouse_receive_packing_lists_enrich.total_package_width_cm])[0]::FLOAT AS total_package_width_cm, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.total_package_height_cm], [origin_warehouse_receive_packing_lists_enrich.total_package_height_cm])[0]::FLOAT AS total_package_height_cm, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.total_package_length_cm], [origin_warehouse_receive_packing_lists_enrich.total_package_length_cm])[0]::FLOAT AS total_package_length_cm, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.total_package_weight_kg], [origin_warehouse_receive_packing_lists_enrich.total_package_weight_kg])[0]::FLOAT AS total_package_weight_kg, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.total_package_item], [origin_warehouse_receive_packing_lists_enrich.total_package_item])[0]::FLOAT AS total_package_item, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.total_package_cbm], [origin_warehouse_receive_packing_lists_enrich.total_package_cbm])[0]::FLOAT AS total_package_cbm, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.number_of_over_weight_charge_record], [origin_warehouse_receive_packing_lists_enrich.number_of_over_weight_charge_record])[0]::INT AS number_of_over_weight_charge_record, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.over_weight_cbm], [origin_warehouse_receive_packing_lists_enrich.over_weight_cbm])[0]::FLOAT AS over_weight_cbm, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.over_weight_kg], [origin_warehouse_receive_packing_lists_enrich.over_weight_kg])[0]::FLOAT AS over_weight_kg, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.number_of_pallet_charge_record], [origin_warehouse_receive_packing_lists_enrich.number_of_pallet_charge_record])[0]::INT AS number_of_pallet_charge_record, + GREATEST([origin_warehouse_shipping_packing_lists_enrich.pallet_charge_cbm], [origin_warehouse_receive_packing_lists_enrich.pallet_charge_cbm])[0]::FLOAT AS pallet_charge_cbm, + + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_arrangement_id, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_transport_mode, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_courier, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_tracking_number, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_status, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_number_of_etd_datetime_postpone, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_number_of_eta_datetime_postpone, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_dispatched_datetime, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_droped_datetime, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_etd_datetimes, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_first_etd_datetime, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_last_etd_datetime, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_eta_datetimes, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_first_eta_datetime, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_last_eta_datetime, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_first_estimate_schedule_created_datetime, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_last_estimate_schedule_created_datetime, + origin_warehouse_receive_packing_lists_enrich.origin_factory_transport_created_datetime, + + + + origin_warehouse_shipping_packing_lists_enrich.origin_warehouse_shipping_packing_list_id AS adjusted_origin_warehouse_shipping_id, + origin_warehouse_shipping_packing_lists_enrich.old_origin_warehouse_shipping_packing_list_id AS origin_warehouse_shipping_id, + origin_warehouse_shipping_packing_lists_enrich.warehouse_sub_company_id AS warehouse_sub_company_id, + origin_warehouse_shipping_packing_lists_enrich.reference_contract, + origin_warehouse_shipping_packing_lists_enrich.origin_warehouse_shipping_order_status AS origin_warehouse_shipping_status, + origin_warehouse_shipping_packing_lists_enrich.origin_warehouse_shipping_order_created_datetime AS origin_warehouse_shipping_created_datetime, + + origin_warehouse_shipping_packing_lists_enrich.invoice_sub_company_id, + origin_warehouse_shipping_packing_lists_enrich.payment_sub_company_id, + origin_warehouse_shipping_packing_lists_enrich.number_of_invoice, + origin_warehouse_shipping_packing_lists_enrich.number_of_payment, + origin_warehouse_shipping_packing_lists_enrich.invoice_number_of_shipping_fee, + origin_warehouse_shipping_packing_lists_enrich.invoice_average_shipping_fee_price_per_quantity, + origin_warehouse_shipping_packing_lists_enrich.invoice_total_shipping_fee_quantity, + origin_warehouse_shipping_packing_lists_enrich.invoice_total_shipping_fee_value_rm, + origin_warehouse_shipping_packing_lists_enrich.invoice_number_of_min_cbm_charge, + origin_warehouse_shipping_packing_lists_enrich.invoice_average_min_cbm_charge_price_per_quantity, + origin_warehouse_shipping_packing_lists_enrich.invoice_total_min_cbm_charge_quantity, + origin_warehouse_shipping_packing_lists_enrich.invoice_total_min_cbm_charge_value_rm, + origin_warehouse_shipping_packing_lists_enrich.invoice_number_of_over_weight_charge, + origin_warehouse_shipping_packing_lists_enrich.invoice_average_over_weight_charge_price_per_quantity, + origin_warehouse_shipping_packing_lists_enrich.invoice_total_over_weight_charge_quantity, + origin_warehouse_shipping_packing_lists_enrich.invoice_total_over_weight_charge_value_rm, + origin_warehouse_shipping_packing_lists_enrich.invoice_number_of_custom_charge, + origin_warehouse_shipping_packing_lists_enrich.invoice_average_custom_charge_price_per_quantity, + origin_warehouse_shipping_packing_lists_enrich.invoice_total_custom_charge_quantity, + origin_warehouse_shipping_packing_lists_enrich.invoice_total_custom_charge_value_rm, + origin_warehouse_shipping_packing_lists_enrich.invoice_value_rm, + origin_warehouse_shipping_packing_lists_enrich.invoice_tax_rm, + origin_warehouse_shipping_packing_lists_enrich.invoice_service_charge, + origin_warehouse_shipping_packing_lists_enrich.payment_value_rm, + origin_warehouse_shipping_packing_lists_enrich.payment_tax_rm, + origin_warehouse_shipping_packing_lists_enrich.payment_service_charge_rm, + origin_warehouse_shipping_packing_lists_enrich.last_invoice_created_datetime, + origin_warehouse_shipping_packing_lists_enrich.last_payment_created_datetime, + + origin_warehouse_shipping_packing_lists_enrich.container_id, + origin_warehouse_shipping_packing_lists_enrich.container_warehouse_sub_company_id, + origin_warehouse_shipping_packing_lists_enrich.container_transport_arrangement_id, + origin_warehouse_shipping_packing_lists_enrich.container_reference, + origin_warehouse_shipping_packing_lists_enrich.container_number, + origin_warehouse_shipping_packing_lists_enrich.container_seal_reference, + origin_warehouse_shipping_packing_lists_enrich.container_type, + origin_warehouse_shipping_packing_lists_enrich.container_status, + origin_warehouse_shipping_packing_lists_enrich.container_transport_arrangment_status, + origin_warehouse_shipping_packing_lists_enrich.container_deliver_courier, + origin_warehouse_shipping_packing_lists_enrich.container_tracking_number, + origin_warehouse_shipping_packing_lists_enrich.number_of_container_etd_datetime_postpone, + origin_warehouse_shipping_packing_lists_enrich.number_of_container_eta_datetime_postpone, + origin_warehouse_shipping_packing_lists_enrich.container_loaded_datetime, + origin_warehouse_shipping_packing_lists_enrich.container_departed_datetime, + origin_warehouse_shipping_packing_lists_enrich.container_arrived_datetime, + origin_warehouse_shipping_packing_lists_enrich.container_etd_datetimes, + origin_warehouse_shipping_packing_lists_enrich.first_container_etd_datetimes, + origin_warehouse_shipping_packing_lists_enrich.last_container_etd_datetimes, + origin_warehouse_shipping_packing_lists_enrich.container_eta_datetimes, + origin_warehouse_shipping_packing_lists_enrich.first_container_eta_datetime, + origin_warehouse_shipping_packing_lists_enrich.last_container_eta_datetime, + origin_warehouse_shipping_packing_lists_enrich.first_container_estimate_schedule_created_datetime, + origin_warehouse_shipping_packing_lists_enrich.last_container_estimate_schedule_created_datetime, + origin_warehouse_shipping_packing_lists_enrich.container_deleted_datetime, + origin_warehouse_shipping_packing_lists_enrich.container_created_datetime, + origin_warehouse_shipping_packing_lists_enrich.container_updated_datetime, + + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_arrangement_id, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_transport_mode, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_courier, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_tracking_number, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_status, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_number_of_etd_datetime_postpone, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_number_of_eta_datetime_postpone, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_dispatched_datetime, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_droped_datetime, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_etd_datetimes, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_first_etd_datetime, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_last_etd_datetime, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_eta_datetimes, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_first_eta_datetime, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_last_eta_datetime, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_first_estimate_schedule_created_datetime, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_last_estimate_schedule_created_datetime, + origin_warehouse_shipping_packing_lists_enrich.destination_warehouse_transport_created_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM origin_warehouse_receive_packing_lists_enrich + + LEFT JOIN origin_warehouse_shipping_packing_lists_enrich + ON (origin_warehouse_receive_packing_lists_enrich.warehouse_order_number = origin_warehouse_shipping_packing_lists_enrich.warehouse_order_number) +), + +-- FINAL +final__fct_shipping__origin_warehouse_receive_packing_lists AS ( + SELECT + -- ids + origin_warehouse_receive_id, + adjusted_origin_warehouse_receive_id, + booking_id, + sub_company_id, + invoice_sub_company_id, + payment_sub_company_id, + warehouse_location_sub_company_id, + warehouse_sub_company_id, + origin_factory_transport_arrangement_id, + origin_warehouse_shipping_id, + adjusted_origin_warehouse_shipping_id, + container_id, + container_warehouse_sub_company_id, + container_transport_arrangement_id, + destination_warehouse_transport_arrangement_id, + + -- dimensions + warehouse_order_number, + has_adjusted_measurment, + service_type, + package_descriptions, + reference_contract, + is_default_delivery_address, + delivery_address_reference, + delivery_address_line_one, + delivery_address_line_two, + delivery_address_district_name, + delivery_address_state_name, + delivery_address_country_name, + delivery_address_postcode, + delivery_address_type, + is_default_billing_address, + billing_address_reference, + billing_address_line_one, + billing_address_line_two, + billing_address_district_name, + billing_address_state_name, + billing_address_country_name, + billing_address_postcode, + billing_address_type, + origin_factory_transport_transport_mode, + origin_factory_transport_courier, + origin_factory_transport_tracking_number, + container_type, + container_reference, + container_number, + container_seal_reference, + container_deliver_courier, + container_tracking_number, + destination_warehouse_transport_transport_mode, + destination_warehouse_transport_courier, + destination_warehouse_transport_tracking_number, + origin_factory_transport_status, + booking_status, + delivery_address_status, + billing_address_status, + origin_warehouse_receive_status, + origin_warehouse_shipping_status, + container_status, + container_transport_arrangment_status, + destination_warehouse_transport_status, + + -- measures + number_of_package_record, + total_package_width_cm, + total_package_height_cm, + total_package_length_cm, + total_package_weight_kg, + total_package_item, + total_package_cbm, + number_of_over_weight_charge_record, + over_weight_cbm, + over_weight_kg, + number_of_pallet_charge_record, + pallet_charge_cbm, + number_of_invoice, + number_of_payment, + invoice_number_of_shipping_fee, + invoice_average_shipping_fee_price_per_quantity, + invoice_total_shipping_fee_quantity, + invoice_total_shipping_fee_value_rm, + invoice_number_of_min_cbm_charge, + invoice_average_min_cbm_charge_price_per_quantity, + invoice_total_min_cbm_charge_quantity, + invoice_total_min_cbm_charge_value_rm, + invoice_number_of_over_weight_charge, + invoice_average_over_weight_charge_price_per_quantity, + invoice_total_over_weight_charge_quantity, + invoice_total_over_weight_charge_value_rm, + invoice_number_of_custom_charge, + invoice_average_custom_charge_price_per_quantity, + invoice_total_custom_charge_quantity, + invoice_total_custom_charge_value_rm, + invoice_value_rm, + invoice_tax_rm, + invoice_service_charge, + payment_value_rm, + payment_tax_rm, + payment_service_charge_rm, + origin_factory_transport_number_of_etd_datetime_postpone, + origin_factory_transport_number_of_eta_datetime_postpone, + number_of_container_etd_datetime_postpone, + number_of_container_eta_datetime_postpone, + destination_warehouse_transport_number_of_etd_datetime_postpone, + destination_warehouse_transport_number_of_eta_datetime_postpone, + + -- date/times + booking_created_datetime, + delivery_address_created_datetime, + billing_address_created_datetime, + origin_factory_transport_created_datetime, + origin_factory_transport_etd_datetimes, + origin_factory_transport_first_etd_datetime, + origin_factory_transport_last_etd_datetime, + origin_factory_transport_dispatched_datetime, + origin_factory_transport_droped_datetime, + origin_factory_transport_eta_datetimes, + origin_factory_transport_first_eta_datetime, + origin_factory_transport_last_eta_datetime, + origin_warehouse_receive_created_datetime, + origin_warehouse_shipping_created_datetime, + last_invoice_created_datetime, + last_payment_created_datetime, + container_created_datetime, + container_loaded_datetime, + container_departed_datetime, + container_arrived_datetime, + container_etd_datetimes, + first_container_etd_datetimes, + last_container_etd_datetimes, + container_eta_datetimes, + first_container_eta_datetime, + last_container_eta_datetime, + destination_warehouse_transport_created_datetime, + destination_warehouse_transport_dispatched_datetime, + destination_warehouse_transport_droped_datetime, + destination_warehouse_transport_etd_datetimes, + destination_warehouse_transport_first_etd_datetime, + destination_warehouse_transport_last_etd_datetime, + destination_warehouse_transport_eta_datetimes, + destination_warehouse_transport_first_eta_datetime, + destination_warehouse_transport_last_eta_datetime, + + -- metadata + _dbt_ran_datetime + + FROM receive_join_shipping_list +) + +SELECT * FROM final__fct_shipping__origin_warehouse_receive_packing_lists diff --git a/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql b/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql new file mode 100644 index 0000000..a0c3738 --- /dev/null +++ b/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql @@ -0,0 +1,327 @@ +-- IMPORT +WITH origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('stg_shipping__origin_warehouse_shipping_packing_lists') }} +), + +bookings AS ( + SELECT * FROM {{ ref('stg_shipping__bookings') }} +), + +booking_delivery_and_billing_addresses AS ( + SELECT * FROM {{ ref('int_shipping__booking_delivery_and_billing_addresses') }} +), + +destination_warehouse_packing_list_transport_arrangements AS ( + SELECT * FROM {{ ref('int_shipping__destination_warehouse_packing_list_transport_arrangements') }} +), + +packing_list_invoice_payments_group_by_packing_list_ids AS ( + SELECT * FROM {{ ref('int_shipping__packing_list_invoice_payments_group_by_packing_list_ids') }} +), + +packages_group_by_packing_list_ids AS ( + SELECT * FROM {{ ref('int_shipping__packages_group_by_packing_list_ids') }} +), + +container_group_by_packing_list_ids AS ( + SELECT * FROM {{ ref('int_shipping__containers_group_by_packing_list_ids') }} +), + + +-- LOGIC +origin_warehouse_shipping_packing_lists_enrich AS ( + SELECT + origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id, + origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id, + origin_warehouse_shipping_packing_lists.has_replaced_by_new_id AS has_replaced_by_new_origin_warehouse_shipping_packing_list_id, + + origin_warehouse_shipping_packing_lists.warehouse_order_number AS warehouse_order_number, + + origin_warehouse_shipping_packing_lists.warehouse_sub_company_id, + origin_warehouse_shipping_packing_lists.reference_contract, + origin_warehouse_shipping_packing_lists.status AS origin_warehouse_shipping_order_status, + origin_warehouse_shipping_packing_lists.created_datetime AS origin_warehouse_shipping_order_created_datetime, + + origin_warehouse_shipping_packing_lists.booking_id, + bookings.sub_company_id, + bookings.service_type, + bookings.status AS booking_status, + bookings.created_datetime AS booking_created_datetime, + + booking_delivery_and_billing_addresses.delivery_address_reference, + booking_delivery_and_billing_addresses.delivery_address_line_one, + booking_delivery_and_billing_addresses.delivery_address_line_two, + booking_delivery_and_billing_addresses.delivery_address_district_name, + booking_delivery_and_billing_addresses.delivery_address_state_name, + booking_delivery_and_billing_addresses.delivery_address_country_name, + booking_delivery_and_billing_addresses.delivery_address_postcode, + booking_delivery_and_billing_addresses.delivery_address_type, + booking_delivery_and_billing_addresses.is_default_delivery_address, + booking_delivery_and_billing_addresses.delivery_address_status, + booking_delivery_and_billing_addresses.billing_address_reference, + booking_delivery_and_billing_addresses.billing_address_line_one, + booking_delivery_and_billing_addresses.billing_address_line_two, + booking_delivery_and_billing_addresses.billing_address_district_name, + booking_delivery_and_billing_addresses.billing_address_state_name, + booking_delivery_and_billing_addresses.billing_address_country_name, + booking_delivery_and_billing_addresses.billing_address_postcode, + booking_delivery_and_billing_addresses.billing_address_type, + booking_delivery_and_billing_addresses.is_default_billing_address, + booking_delivery_and_billing_addresses.billing_address_status, + booking_delivery_and_billing_addresses.delivery_address_created_datetime, + booking_delivery_and_billing_addresses.billing_address_created_datetime, + + packing_list_invoice_payments_group_by_packing_list_ids.invoice_sub_company_id, + packing_list_invoice_payments_group_by_packing_list_ids.payment_sub_company_id, + packing_list_invoice_payments_group_by_packing_list_ids.number_of_invoice, + packing_list_invoice_payments_group_by_packing_list_ids.number_of_payment, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_number_of_shipping_fee, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_average_shipping_fee_price_per_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_shipping_fee_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_shipping_fee_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_number_of_min_cbm_charge, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_average_min_cbm_charge_price_per_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_min_cbm_charge_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_min_cbm_charge_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_number_of_over_weight_charge, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_average_over_weight_charge_price_per_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_over_weight_charge_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_over_weight_charge_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_number_of_custom_charge, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_average_custom_charge_price_per_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_custom_charge_quantity, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_total_custom_charge_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_tax_rm, + packing_list_invoice_payments_group_by_packing_list_ids.invoice_service_charge, + packing_list_invoice_payments_group_by_packing_list_ids.payment_value_rm, + packing_list_invoice_payments_group_by_packing_list_ids.payment_tax_rm, + packing_list_invoice_payments_group_by_packing_list_ids.payment_service_charge_rm, + packing_list_invoice_payments_group_by_packing_list_ids.last_invoice_created_datetime, + packing_list_invoice_payments_group_by_packing_list_ids.last_payment_created_datetime, + + packages_group_by_packing_list_ids.package_descriptions, + packages_group_by_packing_list_ids.number_of_package_record, + packages_group_by_packing_list_ids.total_package_width_cm, + packages_group_by_packing_list_ids.total_package_height_cm, + packages_group_by_packing_list_ids.total_package_length_cm, + packages_group_by_packing_list_ids.total_package_weight_kg, + packages_group_by_packing_list_ids.total_package_item, + packages_group_by_packing_list_ids.total_package_cbm, + packages_group_by_packing_list_ids.number_of_over_weight_charge_record, + packages_group_by_packing_list_ids.over_weight_cbm, + packages_group_by_packing_list_ids.over_weight_kg, + packages_group_by_packing_list_ids.number_of_pallet_charge_record, + packages_group_by_packing_list_ids.pallet_charge_cbm, + + container_group_by_packing_list_ids.container_id, + container_group_by_packing_list_ids.container_warehouse_sub_company_id, + container_group_by_packing_list_ids.container_transport_arrangement_id, + container_group_by_packing_list_ids.container_reference, + container_group_by_packing_list_ids.container_number, + container_group_by_packing_list_ids.container_seal_reference, + container_group_by_packing_list_ids.container_type, + container_group_by_packing_list_ids.container_status, + container_group_by_packing_list_ids.container_transport_arrangment_status, + container_group_by_packing_list_ids.container_deliver_courier, + container_group_by_packing_list_ids.container_tracking_number, + container_group_by_packing_list_ids.number_of_container_etd_datetime_postpone, + container_group_by_packing_list_ids.number_of_container_eta_datetime_postpone, + container_group_by_packing_list_ids.container_loaded_datetime, + container_group_by_packing_list_ids.container_departed_datetime, + container_group_by_packing_list_ids.container_arrived_datetime, + container_group_by_packing_list_ids.container_etd_datetimes, + container_group_by_packing_list_ids.first_container_etd_datetimes, + container_group_by_packing_list_ids.last_container_etd_datetimes, + container_group_by_packing_list_ids.container_eta_datetimes, + container_group_by_packing_list_ids.first_container_eta_datetime, + container_group_by_packing_list_ids.last_container_eta_datetime, + container_group_by_packing_list_ids.first_container_estimate_schedule_created_datetime, + container_group_by_packing_list_ids.last_container_estimate_schedule_created_datetime, + container_group_by_packing_list_ids.container_deleted_datetime, + container_group_by_packing_list_ids.container_created_datetime, + container_group_by_packing_list_ids.container_updated_datetime, + + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_arrangement_id, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_transport_mode, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_courier, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_tracking_number, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_status, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_number_of_etd_datetime_postpone, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_number_of_eta_datetime_postpone, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_dispatched_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_droped_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_etd_datetimes, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_first_etd_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_last_etd_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_eta_datetimes, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_first_eta_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_last_eta_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_first_estimate_schedule_created_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_last_estimate_schedule_created_datetime, + destination_warehouse_packing_list_transport_arrangements.destination_warehouse_transport_created_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN bookings + ON (origin_warehouse_shipping_packing_lists.booking_id = bookings.booking_id) + + LEFT JOIN booking_delivery_and_billing_addresses + ON (bookings.booking_id = booking_delivery_and_billing_addresses.booking_id) + + LEFT JOIN packing_list_invoice_payments_group_by_packing_list_ids + ON (origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = packing_list_invoice_payments_group_by_packing_list_ids.old_origin_warehouse_shipping_packing_list_id) + + LEFT JOIN packages_group_by_packing_list_ids + ON (origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id = packages_group_by_packing_list_ids.packing_list_id) + + LEFT JOIN container_group_by_packing_list_ids + ON (origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = container_group_by_packing_list_ids.packing_list_id) + + LEFT JOIN destination_warehouse_packing_list_transport_arrangements + ON (origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id = destination_warehouse_packing_list_transport_arrangements.old_origin_warehouse_shipping_packing_list_id) +), + + +-- FINAL +final__fct_shipping__origin_warehouse_shipping_packing_lists AS ( + SELECT + -- ids + origin_warehouse_shipping_packing_list_id, + old_origin_warehouse_shipping_packing_list_id, + booking_id, + sub_company_id, + warehouse_sub_company_id, + invoice_sub_company_id, + payment_sub_company_id, + container_id, + container_warehouse_sub_company_id, + container_transport_arrangement_id, + destination_warehouse_transport_arrangement_id, + + -- dimensions + warehouse_order_number, + reference_contract, + has_replaced_by_new_origin_warehouse_shipping_packing_list_id, + service_type, + package_descriptions, + delivery_address_reference, + delivery_address_line_one, + delivery_address_line_two, + delivery_address_district_name, + delivery_address_state_name, + delivery_address_country_name, + delivery_address_postcode, + delivery_address_type, + is_default_delivery_address, + delivery_address_status, + billing_address_reference, + billing_address_line_one, + billing_address_line_two, + billing_address_district_name, + billing_address_state_name, + billing_address_country_name, + billing_address_postcode, + billing_address_type, + is_default_billing_address, + container_reference, + container_number, + container_seal_reference, + container_type, + container_deliver_courier, + container_tracking_number, + destination_warehouse_transport_transport_mode, + destination_warehouse_transport_courier, + destination_warehouse_transport_tracking_number, + booking_status, + billing_address_status, + origin_warehouse_shipping_order_status, + container_status, + container_transport_arrangment_status, + destination_warehouse_transport_status, + + -- measures + number_of_package_record, + total_package_width_cm, + total_package_height_cm, + total_package_length_cm, + total_package_weight_kg, + total_package_item, + total_package_cbm, + number_of_over_weight_charge_record, + over_weight_cbm, + over_weight_kg, + number_of_pallet_charge_record, + pallet_charge_cbm, + number_of_invoice, + number_of_payment, + invoice_number_of_shipping_fee, + invoice_average_shipping_fee_price_per_quantity, + invoice_total_shipping_fee_quantity, + invoice_total_shipping_fee_value_rm, + invoice_number_of_min_cbm_charge, + invoice_average_min_cbm_charge_price_per_quantity, + invoice_total_min_cbm_charge_quantity, + invoice_total_min_cbm_charge_value_rm, + invoice_number_of_over_weight_charge, + invoice_average_over_weight_charge_price_per_quantity, + invoice_total_over_weight_charge_quantity, + invoice_total_over_weight_charge_value_rm, + invoice_number_of_custom_charge, + invoice_average_custom_charge_price_per_quantity, + invoice_total_custom_charge_quantity, + invoice_total_custom_charge_value_rm, + invoice_value_rm, + invoice_tax_rm, + invoice_service_charge, + payment_value_rm, + payment_tax_rm, + payment_service_charge_rm, + number_of_container_etd_datetime_postpone, + number_of_container_eta_datetime_postpone, + destination_warehouse_transport_number_of_etd_datetime_postpone, + destination_warehouse_transport_number_of_eta_datetime_postpone, + + -- date/times + booking_created_datetime, + delivery_address_created_datetime, + billing_address_created_datetime, + origin_warehouse_shipping_order_created_datetime, + last_invoice_created_datetime, + last_payment_created_datetime, + container_loaded_datetime, + container_departed_datetime, + container_arrived_datetime, + container_etd_datetimes, + first_container_etd_datetimes, + last_container_etd_datetimes, + container_eta_datetimes, + first_container_eta_datetime, + last_container_eta_datetime, + first_container_estimate_schedule_created_datetime, + last_container_estimate_schedule_created_datetime, + container_deleted_datetime, + container_created_datetime, + container_updated_datetime, + destination_warehouse_transport_dispatched_datetime, + destination_warehouse_transport_droped_datetime, + destination_warehouse_transport_etd_datetimes, + destination_warehouse_transport_first_etd_datetime, + destination_warehouse_transport_last_etd_datetime, + destination_warehouse_transport_eta_datetimes, + destination_warehouse_transport_first_eta_datetime, + destination_warehouse_transport_last_eta_datetime, + destination_warehouse_transport_first_estimate_schedule_created_datetime, + destination_warehouse_transport_last_estimate_schedule_created_datetime, + destination_warehouse_transport_created_datetime, + + -- metadata + _dbt_ran_datetime + + FROM origin_warehouse_shipping_packing_lists_enrich +) + +SELECT * FROM final__fct_shipping__origin_warehouse_shipping_packing_lists \ No newline at end of file diff --git a/models/staging/shipping/_shipping__docs.md b/models/staging/shipping/_shipping__docs.md index 7535a29..9315aa2 100644 --- a/models/staging/shipping/_shipping__docs.md +++ b/models/staging/shipping/_shipping__docs.md @@ -4,6 +4,8 @@ Application link: https://izyim.cief-malaysia.com/ Data is extract from CIEF Selfhost MYSQL `portal_production`, using custom python code. +- The shipping data is only start at **SEP-2022** + - `_source_loaded_at` will add into the raw data when extract - Python will run the SQL in everyday 12:30AM diff --git a/models/staging/shipping/_shipping__models.yml b/models/staging/shipping/_shipping__models.yml new file mode 100644 index 0000000..62b502f --- /dev/null +++ b/models/staging/shipping/_shipping__models.yml @@ -0,0 +1,17 @@ +version: 2 + +models: + - name: stg_packages + description: use in Packing_List to know what packages is in a PackingList, 1 PackingList can have many Packages. Document is only use by partner HWave + columns: + - name: package_type + description: There are only 3 type now. Future PALLET might be use + tests: + - accepted_values: + values: ['CARTON','OVER_WEIGHT','DOCUMENT'] + + - name: stg_shipping__origin_warehouse_shipping_packing_lists + description: must be tally with receive list + + - name: stg_shipping__packing_list_invoice_payments + description: the invoice & payments data only start record from 2022-09-22 \ No newline at end of file diff --git a/models/staging/shipping/_shipping__sources.yml b/models/staging/shipping/_shipping__sources.yml index 5fcb7a7..1a3acea 100644 --- a/models/staging/shipping/_shipping__sources.yml +++ b/models/staging/shipping/_shipping__sources.yml @@ -17,7 +17,7 @@ sources: owner: "@yam" model_maturity: prod tags: - - exchange + - shipping - daily @@ -92,7 +92,7 @@ sources: - name: company_modules - description: Raw `company_modules` table from Snowflake + description: Also call sub_company in DBT. Companies will be the parents for company_modules columns: - name: id description: Primary key for 'company_modules' @@ -100,6 +100,34 @@ sources: - unique - not_null + - name: company_id + description: Foreign key for 'companies', also known as parent_companies in DBT + tests: + - not_null + + - name: name + description: Company name / Warehouse name etc... + tests: + - not_null + + - name: type + description: When writing the test case, there is only 5 type of sub_company + tests: + - not_null + - accepted_values: + values: [8,7,3,2,1] + + - name: deleted_at + tests: + - dbt_expectations.expect_column_values_to_be_null + + - name: created_at + tests: + - not_null + + - name: updated_at + tests: + - not_null - name: connection_segments description: Raw `connection_segments` table from Snowflake @@ -141,6 +169,26 @@ sources: - not_null + - name: countries + description: Raw `countries` table from Snowflake + columns: + - name: id + description: Primary key for 'countries' + tests: + - unique + - not_null + + + - name: districts + description: Raw `districts` table from Snowflake + columns: + - name: id + description: Primary key for 'districts' + tests: + - unique + - not_null + + - name: documents description: Raw `documents` table from Snowflake columns: @@ -199,6 +247,11 @@ sources: tests: - unique - not_null + + - name: reference + tests: + - dbt_expectations.expect_column_values_to_be_null + - name: packing_lists @@ -209,6 +262,9 @@ sources: tests: - unique - not_null + + - name: owner_type + description: If owner_type is 'CompanyModule', then it mean unclaim parcel which cant found owner. After owner found, it will update owner_type to 'Order' and owner_id. - name: password_resets @@ -242,7 +298,7 @@ sources: - name: segment_constants - description: Raw `segment_constants` table from Snowflake + description: It store the shipping rate / CBM and other rate as well in the value cell columns: - name: id description: Primary key for 'segment_constants' @@ -251,6 +307,16 @@ sources: - not_null + - name: states + description: State id + columns: + - name: id + description: Primary key for 'states' + tests: + - unique + - not_null + + - name: transaction_details description: Raw `transaction_details` table from Snowflake columns: @@ -271,6 +337,16 @@ sources: - not_null + - name: transaction_logs + description: Raw `transaction_logs` table from Snowflake + columns: + - name: id + description: Primary key for 'transaction_logs' + tests: + - unique + - not_null + + - name: transport_arrangements description: Raw `transport_arrangements` table from Snowflake columns: diff --git a/models/staging/shipping/base/base_shipping__addresses.sql b/models/staging/shipping/base/base_shipping__addresses.sql new file mode 100644 index 0000000..ed43355 --- /dev/null +++ b/models/staging/shipping/base/base_shipping__addresses.sql @@ -0,0 +1,65 @@ +-- IMPORTS +WITH addresses AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'addresses') }} +), + +-- LOGIC +addresses_rename AS ( + + SELECT + addresses.id AS address_id, + addresses.owner_type, + addresses.owner_id, + addresses.reference AS address_reference, + addresses.street_one AS address_line_one, + addresses.street_two AS address_line_two, + addresses.district_id, + addresses.state_id, + addresses.country_id, + addresses.postcode, + addresses.type AS address_type, + addresses.default AS is_default_address, + addresses.status, + addresses.deleted_at AS deleted_datetime, + addresses.created_at AS created_datetime, + addresses.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM addresses +), + +-- FINAL +final__base_shipping__addresses AS ( + + SELECT + -- ids + address_id, + owner_id, + district_id, + state_id, + country_id, + + -- dimensions + owner_type, + address_reference, + address_line_one, + address_line_two, + postcode, + address_type, + is_default_address, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM addresses_rename +) + +SELECT * FROM addresses_rename \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__bookings.sql b/models/staging/shipping/base/base_shipping__bookings.sql index fff9113..eb3ca7f 100644 --- a/models/staging/shipping/base/base_shipping__bookings.sql +++ b/models/staging/shipping/base/base_shipping__bookings.sql @@ -20,7 +20,6 @@ orders_rename AS ( '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM orders - ), diff --git a/models/staging/shipping/base/base_shipping__containers.sql b/models/staging/shipping/base/base_shipping__containers.sql new file mode 100644 index 0000000..83c9fe1 --- /dev/null +++ b/models/staging/shipping/base/base_shipping__containers.sql @@ -0,0 +1,60 @@ +-- IMPORT +WITH containers AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'containers') }} +), + +-- LOGIC +containers_rename AS ( + SELECT + id AS container_id, + owner_type, + owner_id, + reference AS container_reference, + container_number, + container_type, + seal_reference, + loading_date AS loading_datetime, + status, + deleted_at AS deleted_datetime, + created_at AS created_datetime, + updated_at AS updated_datetime + + FROM containers +), + +containers_remove_duplicate AS ( + SELECT + *, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM containers_rename +), + +-- FINAL +final__base_shipping__containers AS ( + SELECT + -- ids + container_id, + owner_id, + + -- dimensions + owner_type, + container_reference, + container_number, + container_type, + seal_reference, + status, + + -- measures + loading_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM containers_remove_duplicate +) + +SELECT * FROM final__base_shipping__containers \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__countries.sql b/models/staging/shipping/base/base_shipping__countries.sql new file mode 100644 index 0000000..2a655ac --- /dev/null +++ b/models/staging/shipping/base/base_shipping__countries.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH countries AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'countries') }} +), + +-- LOGIC +countries_rename AS ( + + SELECT + countries.id AS country_id, + countries.name, + countries.short_code, + countries.phone_code, + countries.deleted_at AS deleted_datetime, + countries.created_at AS created_datetime, + countries.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM countries + +), + + +-- FINAL +final__base_shipping__countries AS ( + + SELECT + -- ids + country_id, + + -- dimensions + name, + short_code, + phone_code, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM countries_rename + +) + +SELECT * FROM final__base_shipping__countries \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__districts.sql b/models/staging/shipping/base/base_shipping__districts.sql new file mode 100644 index 0000000..9677d4e --- /dev/null +++ b/models/staging/shipping/base/base_shipping__districts.sql @@ -0,0 +1,53 @@ +-- IMPORTS +WITH districts AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'districts') }} +), + + +-- LOGIC +districts_rename AS ( + + SELECT + districts.id AS district_id, + districts.country_id, + districts.state_id, + districts.name, + districts.postcode, + districts.status, + districts.deleted_at AS deleted_datetime, + districts.created_at AS created_datetime, + districts.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM districts +), + + +-- FINAL +final__base_shipping__districts AS ( + + SELECT + -- ids + district_id, + country_id, + state_id, + + -- dimensions + name, + postcode, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM districts_rename +) + +SELECT * FROM final__base_shipping__districts \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__documents.sql b/models/staging/shipping/base/base_shipping__documents.sql new file mode 100644 index 0000000..2825640 --- /dev/null +++ b/models/staging/shipping/base/base_shipping__documents.sql @@ -0,0 +1,59 @@ +-- IMPORTS +WITH documents AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'documents') }} +), + + +-- LOGIC +documents_rename AS ( + + SELECT + id AS document_id, + owner_type, + owner_id, + document_type, + reference, + status, + issued_date AS issued_datetime, + expired_date AS expired_datetime, + approver AS approver_user_id, + approval_date AS approved_datetime, + deleted_at AS deleted_datetime, + created_at AS created_datetime, + updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM documents +), + +-- FINAL +final__base_shipping__documents AS ( + SELECT + -- ids + document_id, + owner_id, + approver_user_id, + + -- dimensions + owner_type, + document_type, + reference, + status, + + -- measures + + -- date/times + issued_datetime, + expired_datetime, + approved_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM documents_rename +) + +SELECT * FROM final__base_shipping__documents \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__packages.sql b/models/staging/shipping/base/base_shipping__packages.sql new file mode 100644 index 0000000..bd86712 --- /dev/null +++ b/models/staging/shipping/base/base_shipping__packages.sql @@ -0,0 +1,67 @@ +-- IMPORTS +WITH packages AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'packages') }} +), + + +-- LOGIC +packages_rename AS ( + + SELECT + packages.id AS package_id, + packages.packing_list_id, + packages.type AS package_type, + packages.reference, + packages.description AS package_description, + packages.width AS width_cm, + packages.height AS height_cm, + packages.length AS length_cm, + packages.weight AS weight_kg, + packages.quantity, + packages.status, + packages.deleted_at AS deleted_datetime, + packages.created_at AS created_datetime, + packages.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM packages + + WHERE + deleted_datetime IS NULL -- If packages update, new package_id will create. The previous old package_id will have deleted_datetime + +), + + +-- FINAL +final__base_shipping__packages AS ( + + SELECT + -- ids + package_id, + packing_list_id, + + -- dimensions + package_type, + reference, + package_description, + status, + + -- measures + width_cm, + height_cm, + length_cm, + weight_kg, + quantity, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM packages_rename +) + +SELECT * FROM final__base_shipping__packages \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__packing_lists.sql b/models/staging/shipping/base/base_shipping__packing_lists.sql new file mode 100644 index 0000000..921fe1d --- /dev/null +++ b/models/staging/shipping/base/base_shipping__packing_lists.sql @@ -0,0 +1,59 @@ +-- IMPORTS +WITH packing_lists AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'packing_lists') }} +), + + +-- LOGIC +packing_lists_rename AS ( + + SELECT + packing_lists.id AS packing_list_id, + packing_lists.owner_type, + packing_lists.owner_id, + packing_lists.claimant_id AS warehouse_sub_company_id, + packing_lists.reference_contract, + packing_lists.reference, + packing_lists.type AS packing_list_type, + packing_lists.status, + packing_lists.deleted_at AS deleted_datetime, + packing_lists.created_at AS created_datetime, + packing_lists.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM packing_lists + +), + + +-- FINAL +final__base_shipping__packing_lists AS ( + + SELECT + -- ids + packing_list_id, + owner_id, + warehouse_sub_company_id, + + -- dimensions + owner_type, + reference_contract, + reference, + packing_list_type, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- meatadata + _dbt_ran_datetime + + FROM packing_lists_rename + +) + +SELECT * FROM final__base_shipping__packing_lists \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__states.sql b/models/staging/shipping/base/base_shipping__states.sql new file mode 100644 index 0000000..bb202f5 --- /dev/null +++ b/models/staging/shipping/base/base_shipping__states.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH states AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'states') }} +), + + +-- LOGIC +states_rename AS ( + + SELECT + states.id AS state_id, + states.country_id, + states.name, + states.status, + states.deleted_at AS deleted_datetime, + states.created_at AS created_datetime, + states.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM states +), + + +-- FINAL +final__base_shipping__states AS ( + + SELECT + -- ids + state_id, + country_id, + + -- dimensions + name, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM states_rename + +) + +SELECT * FROM final__base_shipping__states \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__transaction_arch_logs.sql b/models/staging/shipping/base/base_shipping__transaction_arch_logs.sql new file mode 100644 index 0000000..7b0944a --- /dev/null +++ b/models/staging/shipping/base/base_shipping__transaction_arch_logs.sql @@ -0,0 +1,78 @@ +-- IMPORTS +WITH transaction_logs AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'transaction_logs') }} +), + + +-- LOGIC +transactions_rename AS ( + SELECT + transaction_logs.id AS transaction_log_id, + transaction_logs.transaction_id, + transaction_logs.owner_type, + transaction_logs.owner_id, + transaction_logs.type AS transaction_type, + transaction_logs.issuer AS issuer_sub_company_id, + transaction_logs.receiver AS receiver_sub_company_id, + transaction_logs.recipient_bank_account_id AS recipient_bank_id, + transaction_logs.payment_method, + transaction_logs.payment_reference, + transaction_logs.bill_no AS bill_number, + transaction_logs.amount AS base_value, + transaction_logs.original_amount AS quote_value, + transaction_logs.currency_id AS base_currency_id, + transaction_logs.original_currency_id AS quote_currency_id, + transaction_logs.currency_rate AS base_to_quote_currency_exchange_rate, + transaction_logs.tax AS base_tax, + transaction_logs.service_charge AS base_service_charge, + transaction_logs.expires_on AS expired_datetime, + transaction_logs.status, + transaction_logs.deleted_at AS deleted_datetime, + transaction_logs.created_at AS created_datetime, + transaction_logs.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_logs +), + +-- FINAL +final__base_shipping__transactions AS ( + SELECT + -- ids + transaction_log_id, + transaction_id, + owner_id, + issuer_sub_company_id, + receiver_sub_company_id, + recipient_bank_id, + base_currency_id, + quote_currency_id, + + -- dimentions + owner_type, + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transactions_rename +) + +SELECT * FROM final__base_shipping__transactions \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__transaction_details.sql b/models/staging/shipping/base/base_shipping__transaction_details.sql new file mode 100644 index 0000000..f903ead --- /dev/null +++ b/models/staging/shipping/base/base_shipping__transaction_details.sql @@ -0,0 +1,52 @@ +-- IMPORTS +WITH transaction_details AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'transaction_details') }} +), + + +-- LOGIC +transaction_details_rename AS ( + SELECT + transaction_details.id AS transaction_detail_id, + transaction_details.transaction_id, + UPPER(transaction_details.reference) AS transaction_detail_type, + transaction_details.name AS transaction_detail_type_description, + transaction_details.quantity, + transaction_details.price AS price_per_quantity, + transaction_details.amount AS base_value, + transaction_details.deleted_at AS deleted_datetime, + transaction_details.created_at AS created_datetime, + transaction_details.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_details +), + +-- FINAL +base_shipping__transaction_details AS ( + SELECT + -- ids + transaction_detail_id, + transaction_id, + + -- dimentions + transaction_detail_type, + transaction_detail_type_description, + + -- measures + quantity, + price_per_quantity, + base_value, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transaction_details_rename +) + +SELECT * FROM base_shipping__transaction_details \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__transactions.sql b/models/staging/shipping/base/base_shipping__transactions.sql new file mode 100644 index 0000000..3c5305a --- /dev/null +++ b/models/staging/shipping/base/base_shipping__transactions.sql @@ -0,0 +1,76 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'transactions') }} +), + + +-- LOGIC +transactions_rename AS ( + SELECT + transactions.id AS transaction_id, + transactions.owner_type, + transactions.owner_id, + transactions.type AS transaction_type, + transactions.issuer AS issuer_sub_company_id, + transactions.receiver AS receiver_sub_company_id, + transactions.recipient_bank_account_id AS recipient_bank_id, + transactions.payment_method, + transactions.payment_reference, + transactions.bill_no AS bill_number, + transactions.amount AS base_value, + transactions.original_amount AS quote_value, + transactions.currency_id AS base_currency_id, + transactions.original_currency_id AS quote_currency_id, + transactions.currency_rate AS base_to_quote_currency_exchange_rate, + transactions.tax AS base_tax, + transactions.service_charge AS base_service_charge, + transactions.expires_on AS expired_datetime, + transactions.status, + transactions.deleted_at AS deleted_datetime, + transactions.created_at AS created_datetime, + transactions.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transactions +), + +-- FINAL +final__base_shipping__transactions AS ( + SELECT + -- ids + transaction_id, + owner_id, + issuer_sub_company_id, + receiver_sub_company_id, + recipient_bank_id, + base_currency_id, + quote_currency_id, + + -- dimentions + owner_type, + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transactions_rename +) + +SELECT * FROM final__base_shipping__transactions \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__transport_arrangements.sql b/models/staging/shipping/base/base_shipping__transport_arrangements.sql new file mode 100644 index 0000000..e7dd339 --- /dev/null +++ b/models/staging/shipping/base/base_shipping__transport_arrangements.sql @@ -0,0 +1,61 @@ +-- IMPORTS +WITH transport_arrangements AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'transport_arrangements') }} +), + + +-- LOGIC +transport_arrangements_rename AS ( + + SELECT + transport_arrangements.id AS transport_arrangement_id, + transport_arrangements.owner_type, + transport_arrangements.owner_id, + transport_arrangements.type AS transport_mode, + transport_arrangements.courier, + transport_arrangements.tracking_number, + transport_arrangements.dispatch_date AS dispatched_datetime, + transport_arrangements.drop_date AS droped_datetime, + transport_arrangements.status, + transport_arrangements.deleted_at AS deleted_datetime, + transport_arrangements.created_at AS created_datetime, + transport_arrangements.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transport_arrangements + +), + + +-- FINAL +final__base_shipping__transport_arrangements AS ( + + SELECT + -- ids + transport_arrangement_id, + owner_id, + + -- dimensions + owner_type, + transport_mode, + courier, + tracking_number, + status, + + -- measures + + -- date/times + dispatched_datetime, + droped_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transport_arrangements_rename + +) + +SELECT * FROM final__base_shipping__transport_arrangements \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__transport_arrangment_estimate_schedules.sql b/models/staging/shipping/base/base_shipping__transport_arrangment_estimate_schedules.sql new file mode 100644 index 0000000..44fb958 --- /dev/null +++ b/models/staging/shipping/base/base_shipping__transport_arrangment_estimate_schedules.sql @@ -0,0 +1,79 @@ +-- IMPORTS +WITH schedules AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'schedules') }} +), + + +-- LOGIC +schedules_rename AS ( + + SELECT + schedules.id AS transport_arrangment_estimate_schedule_id, + schedules.owner_type, + schedules.owner_id, + schedules.etd AS etd_datetime, + schedules.eta AS eta_datetime, + schedules.status, + schedules.deleted_at AS deleted_datetime, + schedules.created_at AS created_datetime, + schedules.updated_at AS updated_datetime + + FROM schedules + +), + +remove_duplicate AS ( + SELECT + MIN(transport_arrangment_estimate_schedule_id) AS transport_arrangment_estimate_schedule_id, + owner_type, + owner_id, + etd_datetime, + eta_datetime, + status, + deleted_datetime, + MIN(created_datetime) AS created_datetime, + MIN(UPDATED_DATETIME) AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + + FROM schedules_rename + + GROUP BY + owner_type, + owner_id, + etd_datetime, + eta_datetime, + status, + deleted_datetime + + ORDER BY created_datetime +), + +-- FINAL +final__base_shipping__transport_arrangment_estimate_schedules AS ( + + SELECT + -- ids + transport_arrangment_estimate_schedule_id, + owner_id, + + -- dimensions + owner_type, + status, + + -- measures + + -- date/times + etd_datetime, + eta_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM remove_duplicate +) + +SELECT * FROM final__base_shipping__transport_arrangment_estimate_schedules \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__booking_addresses.sql b/models/staging/shipping/stg_shipping__booking_addresses.sql new file mode 100644 index 0000000..c8c6edc --- /dev/null +++ b/models/staging/shipping/stg_shipping__booking_addresses.sql @@ -0,0 +1,111 @@ +-- IMPORT +WITH addresses AS ( + SELECT * FROM {{ ref('base_shipping__addresses') }} +), + +countries AS ( + SELECT * FROM {{ ref('base_shipping__countries') }} +), + +states AS ( + SELECT * FROM {{ ref('base_shipping__states') }} +), + +districts AS ( + SELECT * FROM {{ ref('base_shipping__districts') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +address_types AS ( + SELECT * FROM {{ ref('seed_shipping__address_types') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +booking_addresses_rename AS ( + SELECT + addresses.address_id AS booking_address_id, + addresses.owner_id AS booking_id, + addresses.address_reference, + addresses.address_line_one, + addresses.address_line_two, + + addresses.district_id, + districts.name AS district_name, + + addresses.state_id, + states.name AS state_name, + + addresses.country_id, + countries.name AS country_name, + + addresses.postcode, + COALESCE(address_types.name, addresses.address_type::string) AS address_type, + addresses.is_default_address, + COALESCE(status.name, addresses.status::string) AS status, + addresses.deleted_datetime, + addresses.created_datetime, + addresses.updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM addresses + + LEFT JOIN countries + ON (addresses.country_id = countries.country_id) + + LEFT JOIN states + ON (addresses.state_id = states.state_id) + + LEFT JOIN districts + ON (addresses.district_id = districts.district_id) + + LEFT JOIN address_types + ON (addresses.address_type = address_types.id) + + LEFT JOIN status + ON (addresses.status = status.id) + + + WHERE + addresses.owner_type = 'App\\Models\\Order' +), + +-- FINAL +final__stg_shipping__booking_delivery_addresses AS ( + SELECT + -- ids + booking_address_id, + booking_id, + district_id, + state_id, + country_id, + + -- dimensions + address_reference, + address_line_one, + address_line_two, + district_name, + state_name, + country_name, + postcode, + address_type, + is_default_address, + status, + + -- measures + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM booking_addresses_rename +) + +SELECT * FROM final__stg_shipping__booking_delivery_addresses \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__bookings.sql b/models/staging/shipping/stg_shipping__bookings.sql index abfbaf9..1ae0591 100644 --- a/models/staging/shipping/stg_shipping__bookings.sql +++ b/models/staging/shipping/stg_shipping__bookings.sql @@ -1,3 +1,5 @@ +-- IT HAVE ORDER/ ADDRESS which they want to shipping to malaysia + -- IMPORT WITH bookings AS ( SELECT * FROM {{ ref('base_shipping__bookings') }} diff --git a/models/staging/shipping/stg_shipping__company_documents.sql b/models/staging/shipping/stg_shipping__company_documents.sql new file mode 100644 index 0000000..adfd2ef --- /dev/null +++ b/models/staging/shipping/stg_shipping__company_documents.sql @@ -0,0 +1,70 @@ +-- IMPORTS +WITH documents AS ( + SELECT * FROM {{ ref('base_shipping__documents') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +documents_join_status AS ( + + SELECT + documents.document_id, + IFF(documents.owner_type = 'App\\Models\\Company', documents.owner_id, null) AS parent_company_id, + documents.document_type, + documents.reference AS identity_number, + COALESCE(status.name, documents.status::string) AS status, + documents.approver_user_id, + documents.issued_datetime, + documents.expired_datetime, + documents.approved_datetime, + documents.deleted_datetime, + documents.created_datetime, + documents.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM documents + + LEFT JOIN status + ON (documents.status = status.id) + + WHERE + owner_type = 'App\\Models\\Company' +), + + +-- FINAL +final__stg_shipping__company_documents AS ( + + SELECT + -- ids + document_id, + parent_company_id, + approver_user_id, + + -- dimensions + status, + document_type, + identity_number, + + -- measures + + -- date/times + issued_datetime, + expired_datetime, + approved_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM documents_join_status + +) + +SELECT * FROM final__stg_shipping__company_documents \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__container_transport_arrangments.sql b/models/staging/shipping/stg_shipping__container_transport_arrangments.sql new file mode 100644 index 0000000..2ec7088 --- /dev/null +++ b/models/staging/shipping/stg_shipping__container_transport_arrangments.sql @@ -0,0 +1,141 @@ +-- IMPORT +WITH transport_arrangements AS ( + SELECT * FROM {{ ref('base_shipping__transport_arrangements') }} +), + +transport_modes AS ( + SELECT * FROM {{ ref('seed_shipping__transport_modes') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +transport_arrangment_estimate_schedules AS ( + SELECT * FROM {{ ref('base_shipping__transport_arrangment_estimate_schedules') }} +), + +-- LOGIC +transport_arrangment_estimate_schedules_group_by_packing_list_id AS ( + SELECT + owner_id AS transport_arrangement_id, + ARRAY_DISTINCT( + ARRAY_AGG(etd_datetime) WITHIN GROUP (ORDER BY created_datetime ASC) + ) AS etd_datetimes, + etd_datetimes[0]::timestamp AS first_etd_datetime, + etd_datetimes[ARRAY_SIZE(etd_datetimes)-1]::timestamp AS last_etd_datetime, + ARRAY_SIZE(etd_datetimes)-1 AS number_of_etd_datetime_postpone, + + ARRAY_DISTINCT( + ARRAY_AGG(eta_datetime) WITHIN GROUP (ORDER BY created_datetime ASC) + ) AS eta_datetimes, + eta_datetimes[0]::timestamp AS first_eta_datetime, + eta_datetimes[ARRAY_SIZE(eta_datetimes)-1]::timestamp AS last_eta_datetime, + ARRAY_SIZE(eta_datetimes)-1 AS number_of_eta_datetime_postpone, + + MIN(created_datetime) AS first_estimate_schedule_created_datetime, + MAX(created_datetime) AS last_estimate_schedule_created_datetime + + FROM transport_arrangment_estimate_schedules + + WHERE + owner_type = 'App\\Models\\Transport' + + GROUP BY + owner_id +), + +transport_arrangements_mapping AS ( + SELECT + transport_arrangements.transport_arrangement_id, + transport_arrangements.owner_id AS container_id, + + COALESCE(transport_modes.name, transport_arrangements.transport_mode::string) AS transport_mode, + transport_arrangements.courier, + transport_arrangements.tracking_number, + COALESCE(status.name, transport_arrangements.status::string) AS status, + transport_arrangements.dispatched_datetime, + transport_arrangements.droped_datetime, + transport_arrangements.deleted_datetime, + transport_arrangements.created_datetime, + transport_arrangements.updated_datetime, + + + transport_arrangment_estimate_schedules_group_by_packing_list_id.etd_datetimes, + transport_arrangment_estimate_schedules_group_by_packing_list_id.first_etd_datetime, + transport_arrangment_estimate_schedules_group_by_packing_list_id.last_etd_datetime, + transport_arrangment_estimate_schedules_group_by_packing_list_id.number_of_etd_datetime_postpone, + transport_arrangment_estimate_schedules_group_by_packing_list_id.eta_datetimes, + transport_arrangment_estimate_schedules_group_by_packing_list_id.first_eta_datetime, + transport_arrangment_estimate_schedules_group_by_packing_list_id.last_eta_datetime, + transport_arrangment_estimate_schedules_group_by_packing_list_id.number_of_eta_datetime_postpone, + transport_arrangment_estimate_schedules_group_by_packing_list_id.first_estimate_schedule_created_datetime, + transport_arrangment_estimate_schedules_group_by_packing_list_id.last_estimate_schedule_created_datetime + + + FROM transport_arrangements + + LEFT JOIN transport_arrangment_estimate_schedules_group_by_packing_list_id + ON (transport_arrangements.transport_arrangement_id = transport_arrangment_estimate_schedules_group_by_packing_list_id.transport_arrangement_id) + + LEFT JOIN transport_modes + ON (transport_arrangements.transport_mode = transport_modes.id) + + LEFT JOIN status + ON (transport_arrangements.status = status.id) + + WHERE + transport_arrangements.owner_type = 'App\\Models\\Container' +), + +transport_arrangements_mapping_remove_deleted_rows AS ( + SELECT + *, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transport_arrangements_mapping + + WHERE deleted_datetime IS NULL +), + +-- FINAL +final__stg_shipping__container_transport_arrangments AS ( + SELECT + -- ids + transport_arrangement_id, + container_id, + + -- dimensions + transport_mode, + courier, + tracking_number, + status, + + -- measures + number_of_etd_datetime_postpone, + number_of_eta_datetime_postpone, + + -- date/times + dispatched_datetime, + droped_datetime, + etd_datetimes, + first_etd_datetime, + last_etd_datetime, + eta_datetimes, + first_eta_datetime, + last_eta_datetime, + first_estimate_schedule_created_datetime, + last_estimate_schedule_created_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transport_arrangements_mapping_remove_deleted_rows +) + +SELECT * FROM final__stg_shipping__container_transport_arrangments \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__containers.sql b/models/staging/shipping/stg_shipping__containers.sql new file mode 100644 index 0000000..bdd6a3a --- /dev/null +++ b/models/staging/shipping/stg_shipping__containers.sql @@ -0,0 +1,68 @@ +-- IMPORT +WITH containers AS ( + SELECT * FROM {{ ref('base_shipping__containers') }} +), + +container_types AS ( + SELECT * FROM {{ ref('seed_shipping__container_types') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +containers_mapping AS ( + SELECT + container_id, + IFF(owner_type = 'App\\Models\\CompanyModule', owner_id, null) AS sub_company_id, + container_reference, + container_number, + COALESCE(container_types.name, containers.container_type::string) AS container_type, + seal_reference, + COALESCE(status.name, containers.status::string) AS status, + loading_datetime AS loaded_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM containers + + LEFT JOIN status + ON (containers.status = status.id) + + LEFT JOIN container_types + ON (containers.container_type = container_types.id) +), + +-- FINAL +final__stg_shipping__containers AS ( + SELECT + -- ids + container_id, + sub_company_id, + + -- dimensions + container_reference, + container_number, + container_type, + seal_reference, + status, + + -- measures + + -- date/times + loaded_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM containers_mapping +) + +SELECT * FROM final__stg_shipping__containers \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__containers_brg_packing_lists.sql b/models/staging/shipping/stg_shipping__containers_brg_packing_lists.sql new file mode 100644 index 0000000..c798fa7 --- /dev/null +++ b/models/staging/shipping/stg_shipping__containers_brg_packing_lists.sql @@ -0,0 +1,57 @@ +-- IMPORTS +WITH container_packing_lists AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'container_packing_lists') }} +), + + +-- LOGIC +container_packing_lists_rename AS ( + + SELECT + container_packing_lists.id AS container_packing_list_id, + container_packing_lists.container_id, + container_packing_lists.packing_list_id, + container_packing_lists.deleted_at AS deleted_datetime, + container_packing_lists.created_at AS created_datetime, + container_packing_lists.updated_at AS updated_datetime + + FROM container_packing_lists +), + +hardcode_remove_one_packing_list_brg_to_multiple_container AS ( + SELECT + *, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM container_packing_lists_rename + -- This 4 container_packing_list_id container multiple container in a pacing_list (172922,273744,336138,383558) + -- Remove the duplicate + + WHERE container_packing_list_id NOT IN (383558,273744) +), + +-- FINAL +final__stg_shipping__containers_brg_packing_lists AS ( + + SELECT + -- ids + container_packing_list_id, + container_id, + packing_list_id, + + -- dimensions + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM hardcode_remove_one_packing_list_brg_to_multiple_container +) + +SELECT * FROM final__stg_shipping__containers_brg_packing_lists \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__origin_warehouse_receive_packing_lists.sql b/models/staging/shipping/stg_shipping__origin_warehouse_receive_packing_lists.sql new file mode 100644 index 0000000..fcba2db --- /dev/null +++ b/models/staging/shipping/stg_shipping__origin_warehouse_receive_packing_lists.sql @@ -0,0 +1,136 @@ +-- IMPORT +WITH packing_lists AS ( + SELECT * FROM {{ ref('base_shipping__packing_lists') }} +), + +packing_list_types AS ( + SELECT * FROM {{ ref('seed_shipping__packing_list_types') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +packing_lists_type_mapping AS ( + SELECT + packing_lists.packing_list_id, + packing_lists.owner_id, + packing_lists.warehouse_sub_company_id, + packing_lists.owner_type, + packing_lists.reference_contract, + packing_lists.reference, + COALESCE(packing_list_types.name, packing_lists.packing_list_type::string) AS packing_list_type, + COALESCE(status.name, packing_lists.status::string) AS status, + packing_lists.deleted_datetime, + packing_lists.created_datetime, + packing_lists.updated_datetime + + FROM packing_lists + + LEFT JOIN packing_list_types + ON (packing_lists.packing_list_type = packing_list_types.id) + + LEFT JOIN status + ON (packing_lists.status = status.id) + +), + +warehouse_receive_lists AS ( + SELECT + packing_list_id, + owner_id AS booking_id, + warehouse_sub_company_id, + reference_contract, + reference, + status, + deleted_datetime, + created_datetime, + updated_datetime + + FROM + packing_lists_type_mapping + + WHERE + packing_list_type = 'WAREHOUSE_RECEIVE_LIST' + AND + owner_type = 'App\\Models\\Order' + AND + deleted_datetime IS NULL +), + +warehouse_receive_list_replications AS ( + SELECT + packing_list_id AS new_packing_list_id, + owner_id AS packing_list_id, + warehouse_sub_company_id, + reference_contract, + reference, + deleted_datetime, + created_datetime, + updated_datetime + + FROM + packing_lists_type_mapping + + WHERE + packing_list_type = 'WAREHOUSE_RECEIVE_LIST_REPLICA' + AND + owner_type = 'App\\Models\\PackingList' + AND + deleted_datetime IS NULL +), + +warehouse_receive_list_replications_cover_warehouse_receive_lists AS ( + SELECT + + COALESCE(warehouse_receive_list_replications.new_packing_list_id, warehouse_receive_lists.packing_list_id) AS origin_warehouse_receive_packing_list_id, + warehouse_receive_lists.booking_id, + warehouse_receive_lists.warehouse_sub_company_id, + warehouse_receive_lists.reference_contract, + warehouse_receive_lists.reference AS warehouse_order_number, + warehouse_receive_lists.status, + warehouse_receive_lists.deleted_datetime, + warehouse_receive_lists.created_datetime, + warehouse_receive_lists.updated_datetime, + + warehouse_receive_lists.packing_list_id AS old_origin_warehouse_receive_packing_list_id, + IFF(warehouse_receive_list_replications.new_packing_list_id IS NULL, 0, 1) AS has_replaced_by_new_id, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM warehouse_receive_lists + + LEFT JOIN warehouse_receive_list_replications + ON (warehouse_receive_lists.packing_list_id = warehouse_receive_list_replications.packing_list_id) +), + +-- FINAL +final__stg_shipping__origin_warehouse_receive_packing_lists AS ( + SELECT + -- ids + origin_warehouse_receive_packing_list_id, + old_origin_warehouse_receive_packing_list_id, + warehouse_order_number, + booking_id, + warehouse_sub_company_id, + + -- dimensions + has_replaced_by_new_id, + reference_contract, + status, + + -- measures + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM warehouse_receive_list_replications_cover_warehouse_receive_lists +) + +SELECT * FROM final__stg_shipping__origin_warehouse_receive_packing_lists \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__origin_warehouse_shipping_packing_lists.sql b/models/staging/shipping/stg_shipping__origin_warehouse_shipping_packing_lists.sql new file mode 100644 index 0000000..ce1fd24 --- /dev/null +++ b/models/staging/shipping/stg_shipping__origin_warehouse_shipping_packing_lists.sql @@ -0,0 +1,136 @@ +-- IMPORT +WITH packing_lists AS ( + SELECT * FROM {{ ref('base_shipping__packing_lists') }} +), + +packing_list_types AS ( + SELECT * FROM {{ ref('seed_shipping__packing_list_types') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +packing_lists_type_mapping AS ( + SELECT + packing_lists.packing_list_id, + packing_lists.owner_id, + packing_lists.warehouse_sub_company_id, + packing_lists.owner_type, + packing_lists.reference_contract, + packing_lists.reference, + COALESCE(packing_list_types.name, packing_lists.packing_list_type::string) AS packing_list_type, + COALESCE(status.name, packing_lists.status::string) AS status, + packing_lists.deleted_datetime, + packing_lists.created_datetime, + packing_lists.updated_datetime + + FROM packing_lists + + LEFT JOIN packing_list_types + ON (packing_lists.packing_list_type = packing_list_types.id) + + LEFT JOIN status + ON (packing_lists.status = status.id) + +), + +warehouse_shipping_lists AS ( + SELECT + packing_list_id, + owner_id AS booking_id, + warehouse_sub_company_id, + reference_contract, + reference, + status, + deleted_datetime, + created_datetime, + updated_datetime + + FROM + packing_lists_type_mapping + + WHERE + packing_list_type = 'SHIPPING_PACKING_LIST' + AND + owner_type = 'App\\Models\\Order' + AND + deleted_datetime IS NULL +), + +warehouse_shipping_list_replications AS ( + SELECT + packing_list_id AS new_packing_list_id, + owner_id AS packing_list_id, + warehouse_sub_company_id, + reference_contract, + reference, + deleted_datetime, + created_datetime, + updated_datetime + + FROM + packing_lists_type_mapping + + WHERE + packing_list_type = 'SHIPPING_PACKING_LIST_REPLICA' + AND + owner_type = 'App\\Models\\PackingList' + AND + deleted_datetime IS NULL +), + +warehouse_shipping_list_replications_cover_warehouse_shipping_lists AS ( + SELECT + + COALESCE(warehouse_shipping_list_replications.new_packing_list_id, warehouse_shipping_lists.packing_list_id) AS origin_warehouse_shipping_packing_list_id, + warehouse_shipping_lists.booking_id, + warehouse_shipping_lists.warehouse_sub_company_id, + warehouse_shipping_lists.reference_contract, + warehouse_shipping_lists.reference AS warehouse_order_number, + warehouse_shipping_lists.status, + warehouse_shipping_lists.deleted_datetime, + warehouse_shipping_lists.created_datetime, + warehouse_shipping_lists.updated_datetime, + + warehouse_shipping_lists.packing_list_id AS old_origin_warehouse_shipping_packing_list_id, + IFF(warehouse_shipping_list_replications.new_packing_list_id IS NULL, 0, 1) AS has_replaced_by_new_id, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM warehouse_shipping_lists + + LEFT JOIN warehouse_shipping_list_replications + ON (warehouse_shipping_lists.packing_list_id = warehouse_shipping_list_replications.packing_list_id) +), + +-- FINAL +final__stg_shipping__origin_warehouse_shipping_packing_lists AS ( + SELECT + -- ids + origin_warehouse_shipping_packing_list_id, + old_origin_warehouse_shipping_packing_list_id, + warehouse_order_number, + booking_id, + warehouse_sub_company_id, + + -- dimensions + has_replaced_by_new_id, + reference_contract, + status, + + -- measures + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM warehouse_shipping_list_replications_cover_warehouse_shipping_lists +) + +SELECT * FROM final__stg_shipping__origin_warehouse_shipping_packing_lists \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__package_extra_charges.sql b/models/staging/shipping/stg_shipping__package_extra_charges.sql new file mode 100644 index 0000000..a7dc24f --- /dev/null +++ b/models/staging/shipping/stg_shipping__package_extra_charges.sql @@ -0,0 +1,87 @@ +-- IMPORT +WITH packages AS ( + SELECT * FROM {{ ref('base_shipping__packages') }} +), + +package_types AS ( + SELECT * FROM {{ ref('seed_shipping__package_types') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +package_mapping AS ( + SELECT + packages.package_id, + packages.packing_list_id, + COALESCE(package_types.name, packages.package_type::string) AS package_type, + packages.reference, + packages.package_description, + COALESCE(status.name, packages.status::string) AS status, + packages.width_cm, + packages.height_cm, + packages.length_cm, + packages.weight_kg, + packages.quantity, + ROUND(((packages.width_cm/100) * (packages.height_cm/100) * (packages.length_cm/100)) * packages.quantity, 5) AS extra_charge_cbm, + IFF(COALESCE(package_types.name, packages.package_type::string) = 'OVER_WEIGHT', ROUND(extra_charge_cbm * 500, 2), null) AS over_weight_kg, + packages.deleted_datetime, + packages.created_datetime, + packages.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + + FROM packages + + LEFT JOIN package_types + ON (packages.package_type = package_types.id) + + LEFT JOIN status + ON (packages.status = status.id) + + WHERE COALESCE(package_types.name, packages.package_type::string) IN ('OVER_WEIGHT' , 'PALLET') -- This is extra charge or additional service etc.. +), + +-- FINAL +final__stg_shipping__packages AS ( + SELECT + -- ids + package_id, + packing_list_id, + + -- dimensions + package_type, + reference, + package_description, + status, + + -- measures + width_cm, + height_cm, + length_cm, + weight_kg, + quantity, + extra_charge_cbm, + over_weight_kg, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM package_mapping +) + +SELECT * FROM final__stg_shipping__packages + + + + + diff --git a/models/staging/shipping/stg_shipping__packages.sql b/models/staging/shipping/stg_shipping__packages.sql new file mode 100644 index 0000000..498300a --- /dev/null +++ b/models/staging/shipping/stg_shipping__packages.sql @@ -0,0 +1,85 @@ +-- IMPORT +WITH packages AS ( + SELECT * FROM {{ ref('base_shipping__packages') }} +), + +package_types AS ( + SELECT * FROM {{ ref('seed_shipping__package_types') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +package_mapping AS ( + SELECT + packages.package_id, + packages.packing_list_id, + COALESCE(package_types.name, packages.package_type::string) AS package_type, + packages.reference, + packages.package_description, + COALESCE(status.name, packages.status::string) AS status, + packages.width_cm, + packages.height_cm, + packages.length_cm, + packages.weight_kg, + packages.quantity, + ROUND(((packages.width_cm/100) * (packages.height_cm/100) * (packages.length_cm/100)) * packages.quantity, 5) AS cbm, + packages.deleted_datetime, + packages.created_datetime, + packages.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + + FROM packages + + LEFT JOIN package_types + ON (packages.package_type = package_types.id) + + LEFT JOIN status + ON (packages.status = status.id) + + WHERE COALESCE(package_types.name, packages.package_type::string) = 'CARTON' -- CARTON means real item that receive, not extra charge or additional service etc.. +), + +-- FINAL +final__stg_shipping__packages AS ( + SELECT + -- ids + package_id, + packing_list_id, + + -- dimensions + package_type, + reference, + package_description, + status, + + -- measures + width_cm, + height_cm, + length_cm, + weight_kg, + quantity, + cbm, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM package_mapping +) + +SELECT * FROM final__stg_shipping__packages + + + + + diff --git a/models/staging/shipping/stg_shipping__packing_list_invoice_payments.sql b/models/staging/shipping/stg_shipping__packing_list_invoice_payments.sql new file mode 100644 index 0000000..c95eeb4 --- /dev/null +++ b/models/staging/shipping/stg_shipping__packing_list_invoice_payments.sql @@ -0,0 +1,263 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_shipping__transactions') }} +), + +transaction_details AS ( + SELECT * FROM {{ ref('base_shipping__transaction_details') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_shipping__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_shipping__payment_methods') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +transactions_mapping AS ( + SELECT + transaction_id, + owner_id, + issuer_sub_company_id, + receiver_sub_company_id, + recipient_bank_id, + base_currency_id, + quote_currency_id, + owner_type, + COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, + COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + COALESCE(status.name, transactions.status::string) AS status, + payment_reference, + bill_number, + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime + + FROM transactions + + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) + + LEFT JOIN status + ON (transactions.status = status.id) +), + +packing_list_invoices_rename AS ( + SELECT + transaction_id AS packing_list_invoice_id, + owner_id AS old_origin_warehouse_shipping_packing_list_id, + receiver_sub_company_id AS invoice_sub_company_id, + status AS invoice_status, + bill_number AS invoice_number, + base_value AS invoice_value_rm, + base_tax AS invoice_tax_rm, + base_service_charge AS invoice_service_charge, + deleted_datetime AS invoice_deleted_datetime, + created_datetime AS invoice_created_datetime, + updated_datetime AS invoice_updated_datetime + + FROM + transactions_mapping + + WHERE + owner_type = 'App\\Models\\PackingList' + AND + transaction_type = 'SHIPPING_INVOICE' + AND + deleted_datetime IS NULL +), + +transaction_details_group_by_packing_list_invoice_id AS ( + SELECT + transaction_id AS packing_list_invoice_id, + NULLIFZERO(COUNT(IFF(transaction_detail_type = 'SHIPPING_FEE', transaction_detail_id, null))) AS invoice_number_of_shipping_fee, + LISTAGG(IFF(transaction_detail_type = 'SHIPPING_FEE', transaction_detail_type_description, null), ', ') AS invoice_shipping_fee_descriptions, + AVG(IFF(transaction_detail_type = 'SHIPPING_FEE', price_per_quantity, null)) AS invoice_average_shipping_fee_price_per_quantity, + SUM(IFF(transaction_detail_type = 'SHIPPING_FEE', quantity, null)) AS invoice_total_shipping_fee_quantity, + SUM(IFF(transaction_detail_type = 'SHIPPING_FEE', base_value, null)) AS invoice_total_shipping_fee_value_rm, + + NULLIFZERO(COUNT(IFF(transaction_detail_type = 'MIN_CBM_CHARGES', transaction_detail_id, null))) AS invoice_number_of_min_cbm_charge, + LISTAGG(IFF(transaction_detail_type = 'MIN_CBM_CHARGES', transaction_detail_type_description, null), ', ') AS invoice_min_cbm_charge_descriptions, + AVG(IFF(transaction_detail_type = 'MIN_CBM_CHARGES', price_per_quantity, null)) AS invoice_average_min_cbm_charge_price_per_quantity, + SUM(IFF(transaction_detail_type = 'MIN_CBM_CHARGES', quantity, null)) AS invoice_total_min_cbm_charge_quantity, + SUM(IFF(transaction_detail_type = 'MIN_CBM_CHARGES', base_value, null)) AS invoice_total_min_cbm_charge_value_rm, + + NULLIFZERO(COUNT(IFF(transaction_detail_type = 'OVER_WEIGHT_CHARGES', transaction_detail_id, null))) AS invoice_number_of_over_weight_charge, + LISTAGG(IFF(transaction_detail_type = 'OVER_WEIGHT_CHARGES', transaction_detail_type_description, null), ', ') AS invoice_over_weight_charge_descriptions, + AVG(IFF(transaction_detail_type = 'OVER_WEIGHT_CHARGES', price_per_quantity, null)) AS invoice_average_over_weight_charge_price_per_quantity, + SUM(IFF(transaction_detail_type = 'OVER_WEIGHT_CHARGES', quantity, null)) AS invoice_total_over_weight_charge_quantity, + SUM(IFF(transaction_detail_type = 'OVER_WEIGHT_CHARGES', base_value, null)) AS invoice_total_over_weight_charge_value_rm, + + NULLIFZERO(COUNT(IFF(transaction_detail_type = 'CUSTOM_CHARGES', transaction_detail_id, null))) AS invoice_number_of_custom_charge, + LISTAGG(IFF(transaction_detail_type = 'CUSTOM_CHARGES', transaction_detail_type_description, null), ', ') AS invoice_custom_charge_descriptions, + AVG(IFF(transaction_detail_type = 'CUSTOM_CHARGES', price_per_quantity, null)) AS invoice_average_custom_charge_price_per_quantity, + SUM(IFF(transaction_detail_type = 'CUSTOM_CHARGES', quantity, null)) AS invoice_total_custom_charge_quantity, + SUM(IFF(transaction_detail_type = 'CUSTOM_CHARGES', base_value, null)) AS invoice_total_custom_charge_value_rm + + FROM transaction_details + + GROUP BY packing_list_invoice_id +), + +payments_rename AS ( + SELECT + transaction_id AS packing_list_payment_id, + owner_id AS packing_list_invoice_id, + issuer_sub_company_id AS payment_sub_company_id, + payment_method, + status AS payment_status, + payment_reference, + bill_number AS payment_number, + base_value AS payment_value_rm, + base_tax AS payment_tax_rm, + base_service_charge AS payment_service_charge_rm, + deleted_datetime AS payment_deleted_datetime, + created_datetime AS payment_created_datetime, + updated_datetime AS payment_updated_datetime + + FROM transactions_mapping + + WHERE + owner_type = 'App\\Models\\Transaction' + AND + transaction_type = 'PAYMENT' + AND + deleted_datetime IS NULL +), + +transaction_packing_lists_enrich AS ( + SELECT + packing_list_invoices_rename.packing_list_invoice_id, + packing_list_invoices_rename.old_origin_warehouse_shipping_packing_list_id, + packing_list_invoices_rename.invoice_sub_company_id, + packing_list_invoices_rename.invoice_status, + packing_list_invoices_rename.invoice_number, + ROUND(packing_list_invoices_rename.invoice_value_rm, 2) AS invoice_value_rm, + packing_list_invoices_rename.invoice_tax_rm, + packing_list_invoices_rename.invoice_service_charge, + packing_list_invoices_rename.invoice_deleted_datetime, + packing_list_invoices_rename.invoice_created_datetime, + packing_list_invoices_rename.invoice_updated_datetime, + + transaction_details_group_by_packing_list_invoice_id.invoice_number_of_shipping_fee, + NULLIF(transaction_details_group_by_packing_list_invoice_id.invoice_shipping_fee_descriptions, '') AS invoice_shipping_fee_descriptions, + transaction_details_group_by_packing_list_invoice_id.invoice_average_shipping_fee_price_per_quantity, + transaction_details_group_by_packing_list_invoice_id.invoice_total_shipping_fee_quantity, + transaction_details_group_by_packing_list_invoice_id.invoice_total_shipping_fee_value_rm, + transaction_details_group_by_packing_list_invoice_id.invoice_number_of_min_cbm_charge, + NULLIF(transaction_details_group_by_packing_list_invoice_id.invoice_min_cbm_charge_descriptions, '') AS invoice_min_cbm_charge_descriptions, + transaction_details_group_by_packing_list_invoice_id.invoice_average_min_cbm_charge_price_per_quantity, + transaction_details_group_by_packing_list_invoice_id.invoice_total_min_cbm_charge_quantity, + transaction_details_group_by_packing_list_invoice_id.invoice_total_min_cbm_charge_value_rm, + transaction_details_group_by_packing_list_invoice_id.invoice_number_of_over_weight_charge, + NULLIF(transaction_details_group_by_packing_list_invoice_id.invoice_over_weight_charge_descriptions, '') AS invoice_over_weight_charge_descriptions, + transaction_details_group_by_packing_list_invoice_id.invoice_average_over_weight_charge_price_per_quantity, + transaction_details_group_by_packing_list_invoice_id.invoice_total_over_weight_charge_quantity, + transaction_details_group_by_packing_list_invoice_id.invoice_total_over_weight_charge_value_rm, + transaction_details_group_by_packing_list_invoice_id.invoice_number_of_custom_charge, + NULLIF(transaction_details_group_by_packing_list_invoice_id.invoice_custom_charge_descriptions, '') AS invoice_custom_charge_descriptions, + transaction_details_group_by_packing_list_invoice_id.invoice_average_custom_charge_price_per_quantity, + transaction_details_group_by_packing_list_invoice_id.invoice_total_custom_charge_quantity, + transaction_details_group_by_packing_list_invoice_id.invoice_total_custom_charge_value_rm, + + payments_rename.packing_list_payment_id, + payments_rename.payment_sub_company_id, + payments_rename.payment_method, + payments_rename.payment_status, + payments_rename.payment_reference, + payments_rename.payment_number, + ROUND(payments_rename.payment_value_rm, 2) AS payment_value_rm, + payments_rename.payment_tax_rm, + payments_rename.payment_service_charge_rm, + payments_rename.payment_deleted_datetime, + payments_rename.payment_created_datetime, + payments_rename.payment_updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM + packing_list_invoices_rename + + LEFT JOIN transaction_details_group_by_packing_list_invoice_id + ON (packing_list_invoices_rename.packing_list_invoice_id = transaction_details_group_by_packing_list_invoice_id.packing_list_invoice_id) + + LEFT JOIN payments_rename + ON (packing_list_invoices_rename.packing_list_invoice_id = payments_rename.packing_list_invoice_id) +), + +final__stg_shipping__packing_list_invoice_payments AS ( + SELECT + -- ids + packing_list_invoice_id, + packing_list_payment_id, + old_origin_warehouse_shipping_packing_list_id, + invoice_sub_company_id, + payment_sub_company_id, + + -- dimensions + invoice_status, + invoice_number, + invoice_shipping_fee_descriptions, + invoice_min_cbm_charge_descriptions, + invoice_over_weight_charge_descriptions, + invoice_custom_charge_descriptions, + payment_method, + payment_status, + payment_reference, + payment_number, + + -- measures + invoice_number_of_shipping_fee, + invoice_average_shipping_fee_price_per_quantity, + invoice_total_shipping_fee_quantity, + invoice_total_shipping_fee_value_rm, + invoice_number_of_min_cbm_charge, + invoice_average_min_cbm_charge_price_per_quantity, + invoice_total_min_cbm_charge_quantity, + invoice_total_min_cbm_charge_value_rm, + invoice_number_of_over_weight_charge, + invoice_average_over_weight_charge_price_per_quantity, + invoice_total_over_weight_charge_quantity, + invoice_total_over_weight_charge_value_rm, + invoice_number_of_custom_charge, + invoice_average_custom_charge_price_per_quantity, + invoice_total_custom_charge_quantity, + invoice_total_custom_charge_value_rm, + invoice_value_rm, + invoice_tax_rm, + invoice_service_charge, + payment_value_rm, + payment_tax_rm, + payment_service_charge_rm, + + -- datet/times + invoice_deleted_datetime, + payment_deleted_datetime, + invoice_created_datetime, + payment_created_datetime, + invoice_updated_datetime, + payment_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transaction_packing_lists_enrich +) + +SELECT * FROM final__stg_shipping__packing_list_invoice_payments \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__packing_list_transport_arrangments.sql b/models/staging/shipping/stg_shipping__packing_list_transport_arrangments.sql new file mode 100644 index 0000000..96954d3 --- /dev/null +++ b/models/staging/shipping/stg_shipping__packing_list_transport_arrangments.sql @@ -0,0 +1,134 @@ +-- IMPORT +WITH transport_arrangements AS ( + SELECT * FROM {{ ref('base_shipping__transport_arrangements') }} +), + +transport_modes AS ( + SELECT * FROM {{ ref('seed_shipping__transport_modes') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +transport_arrangment_estimate_schedules AS ( + SELECT * FROM {{ ref('base_shipping__transport_arrangment_estimate_schedules') }} +), + +-- LOGIC +transport_arrangment_estimate_schedules_group_by_transport_arrangement_id AS ( + SELECT + owner_id AS transport_arrangement_id, + ARRAY_DISTINCT( + ARRAY_AGG(etd_datetime) WITHIN GROUP (ORDER BY created_datetime ASC) + ) AS etd_datetimes, + etd_datetimes[0]::timestamp AS first_etd_datetime, + etd_datetimes[ARRAY_SIZE(etd_datetimes)-1]::timestamp AS last_etd_datetime, + ARRAY_SIZE(etd_datetimes)-1 AS number_of_etd_datetime_postpone, + + ARRAY_DISTINCT( + ARRAY_AGG(eta_datetime) WITHIN GROUP (ORDER BY created_datetime ASC) + ) AS eta_datetimes, + eta_datetimes[0]::timestamp AS first_eta_datetime, + eta_datetimes[ARRAY_SIZE(eta_datetimes)-1]::timestamp AS last_eta_datetime, + ARRAY_SIZE(eta_datetimes)-1 AS number_of_eta_datetime_postpone, + + MIN(created_datetime) AS first_estimate_schedule_created_datetime, + MAX(created_datetime) AS last_estimate_schedule_created_datetime + + FROM transport_arrangment_estimate_schedules + + WHERE + owner_type = 'App\\Models\\Transport' + + GROUP BY + owner_id +), + + +transport_arrangements_mapping AS ( + SELECT + transport_arrangements.transport_arrangement_id, + transport_arrangements.owner_id AS packing_list_id, + + COALESCE(transport_modes.name, transport_arrangements.transport_mode::string) AS transport_mode, + transport_arrangements.courier, + transport_arrangements.tracking_number, + COALESCE(status.name, transport_arrangements.status::string) AS status, + transport_arrangements.dispatched_datetime, + transport_arrangements.droped_datetime, + transport_arrangements.created_datetime, + transport_arrangements.updated_datetime, + + transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.etd_datetimes, + transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.first_etd_datetime, + transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.last_etd_datetime, + transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.number_of_etd_datetime_postpone, + transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.eta_datetimes, + transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.first_eta_datetime, + transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.last_eta_datetime, + transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.number_of_eta_datetime_postpone, + transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.first_estimate_schedule_created_datetime, + transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.last_estimate_schedule_created_datetime, + + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + + FROM transport_arrangements + + LEFT JOIN transport_arrangment_estimate_schedules_group_by_transport_arrangement_id + ON (transport_arrangements.transport_arrangement_id = transport_arrangment_estimate_schedules_group_by_transport_arrangement_id.transport_arrangement_id) + + LEFT JOIN transport_modes + ON (transport_arrangements.transport_mode = transport_modes.id) + + LEFT JOIN status + ON (transport_arrangements.status = status.id) + + WHERE + transport_arrangements.owner_type = 'App\\Models\\PackingList' + AND + transport_arrangements.deleted_datetime IS NULL +), + +-- FINAL +final__stg_shipping__packing_list_transport_arrangments AS ( + SELECT + -- ids + transport_arrangement_id, + packing_list_id, + + -- dimensions + transport_mode, + courier, + tracking_number, + status, + + -- measures + number_of_etd_datetime_postpone, + number_of_eta_datetime_postpone, + + -- date/times + dispatched_datetime, + droped_datetime, + etd_datetimes, + first_etd_datetime, + last_etd_datetime, + eta_datetimes, + first_eta_datetime, + last_eta_datetime, + first_estimate_schedule_created_datetime, + last_estimate_schedule_created_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transport_arrangements_mapping +) + +SELECT * FROM final__stg_shipping__packing_list_transport_arrangments \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__sub_company_addresses.sql b/models/staging/shipping/stg_shipping__sub_company_addresses.sql new file mode 100644 index 0000000..40d8365 --- /dev/null +++ b/models/staging/shipping/stg_shipping__sub_company_addresses.sql @@ -0,0 +1,112 @@ +-- IMPORT +WITH addresses AS ( + SELECT * FROM {{ ref('base_shipping__addresses') }} +), + +countries AS ( + SELECT * FROM {{ ref('base_shipping__countries') }} +), + +states AS ( + SELECT * FROM {{ ref('base_shipping__states') }} +), + +districts AS ( + SELECT * FROM {{ ref('base_shipping__districts') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +address_types AS ( + SELECT * FROM {{ ref('seed_shipping__address_types') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +sub_company_addresses_rename AS ( + SELECT + addresses.address_id AS sub_company_address_id, + addresses.owner_id AS sub_company_id, + addresses.address_reference, + addresses.address_line_one, + addresses.address_line_two, + + addresses.district_id, + districts.name AS district_name, + + addresses.state_id, + states.name AS state_name, + + addresses.country_id, + countries.name AS country_name, + + addresses.postcode, + COALESCE(address_types.name, addresses.address_type::string) AS address_type, + addresses.is_default_address, + COALESCE(status.name, addresses.status::string) AS status, + addresses.deleted_datetime, + addresses.created_datetime, + addresses.updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM addresses + + LEFT JOIN countries + ON (addresses.country_id = countries.country_id) + + LEFT JOIN states + ON (addresses.state_id = states.state_id) + + LEFT JOIN districts + ON (addresses.district_id = districts.district_id) + + LEFT JOIN address_types + ON (addresses.address_type = address_types.id) + + LEFT JOIN status + ON (addresses.status = status.id) + + + WHERE + addresses.owner_type = 'App\\Models\\CompanyModule' + +), + +-- FINAL +final__stg_shipping__sub_company_addresses AS ( + SELECT + -- ids + sub_company_address_id, + sub_company_id, + district_id, + state_id, + country_id, + + -- dimensions + address_reference, + address_line_one, + address_line_two, + district_name, + state_name, + country_name, + postcode, + address_type, + is_default_address, + status, + + -- measures + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM sub_company_addresses_rename +) + +SELECT * FROM final__stg_shipping__sub_company_addresses \ No newline at end of file diff --git a/packages.yml b/packages.yml index 15f3370..a87b53f 100644 --- a/packages.yml +++ b/packages.yml @@ -17,4 +17,7 @@ packages: version: 0.8.2 - package: dbt-labs/dbt_utils - version: 1.0.0 \ No newline at end of file + version: 1.0.0 + + - package: dbt-labs/codegen + version: 0.9.0 \ No newline at end of file diff --git a/seeds/shipping/seed_shipping__container_types.csv b/seeds/shipping/seed_shipping__container_types.csv new file mode 100644 index 0000000..685be47 --- /dev/null +++ b/seeds/shipping/seed_shipping__container_types.csv @@ -0,0 +1,8 @@ +ID,CATEGORY,NAME +1,DEFAULT,TWENTY_FEET_DRY_CONTAINER +2,DEFAULT,FORTY_FEET_DRY_CONTAINER +3,DEFAULT,FORTY_FEET_HIGH_CUBE_CONTAINER +4,DEFAULT,FORTY_FIVE_FEET_HIGH_CUBE_CONTAINER + + + diff --git a/seeds/shipping/seed_shipping__package_types.csv b/seeds/shipping/seed_shipping__package_types.csv new file mode 100644 index 0000000..ca91433 --- /dev/null +++ b/seeds/shipping/seed_shipping__package_types.csv @@ -0,0 +1,5 @@ +ID,CATEGORY,NAME +0,DEFAULT,CARTON +1,DEFAULT,PALLET +2,DEFAULT,OVER_WEIGHT +3,DEFAULT,DOCUMENT \ No newline at end of file diff --git a/seeds/shipping/seed_shipping__payment_methods.csv b/seeds/shipping/seed_shipping__payment_methods.csv new file mode 100644 index 0000000..7726ae0 --- /dev/null +++ b/seeds/shipping/seed_shipping__payment_methods.csv @@ -0,0 +1,6 @@ +ID,CATEGORY,NAME +1,DEFAULT,CASH +2,DEFAULT,CHEQUE +3,DEFAULT,BA +4,DEFAULT,WALLET +5,DEFAULT,PAYMENT_GATEWAY \ No newline at end of file diff --git a/seeds/shipping/seed_shipping__transaction_types.csv b/seeds/shipping/seed_shipping__transaction_types.csv new file mode 100644 index 0000000..9614a7c --- /dev/null +++ b/seeds/shipping/seed_shipping__transaction_types.csv @@ -0,0 +1,17 @@ +ID,CATEGORY,NAME +0,DEFAULT,PAYMENT_ATTEMPT +1,DEFAULT,SHIPPING_INVOICE +2,DEFAULT,PAYMENT +3,DEFAULT,BILL +4,DEFAULT,PERFORMA +5,DEFAULT,TOP_UP +6,DEFAULT,REFUND +7,DEFAULT,PURCHASE_ORDER +8,DEFAULT,SUPPLIER_DELIVER +9,DEFAULT,CREDIT_NOTE +10,DEFAULT,WITHDRAW +11,DEFAULT,DEBIT_NOTE +12,DEFAULT,TRANSFER_FEE +13,DEFAULT,CASH_BACK +14,DEFAULT,SHIPPING_COST +15,DEFAULT,GROUP_PAYMENT \ No newline at end of file diff --git a/seeds/shipping/seed_shipping__transport_modes.csv b/seeds/shipping/seed_shipping__transport_modes.csv new file mode 100644 index 0000000..c3b280a --- /dev/null +++ b/seeds/shipping/seed_shipping__transport_modes.csv @@ -0,0 +1,4 @@ +ID,CATEGORY,NAME +0,DEFAULT,AIR +1,DEFAULT,SEA +2,DEFAULT,LAND \ No newline at end of file