DBT structure change

This commit is contained in:
Yam ZhengLim
2023-05-09 09:53:28 +00:00
parent b3373e5b58
commit 2d2501cbfe
317 changed files with 7448 additions and 10782 deletions
-3
View File
@@ -1,3 +0,0 @@
SELECT distinct QUOTE_CURRENCY_ID ,QUOTE_CURRENCY_NAME
FROM {{ ref('fct_exchange__bookings') }}
-341
View File
@@ -1,341 +0,0 @@
-- MARCRO for count the cummulative_activity_number
{% set funnel_activity = ['register','email_verified','identity_documents_uploaded','identity_documents_verified','first_time_booking','first_time_order','first_time_order_completed','repeat_completed_order_after_30days','repeat_completed_order_after_60days']%}
-- IMPORT
WITH companies AS (
SELECT * FROM {{ ref('dim_exchange__companies') }}
),
transaction_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
company_identity_documents AS (
SELECT * FROM {{ ref('fct_exchange__company_identity_documents') }}
),
email_verifications AS (
SELECT * FROM {{ ref('fct_exchange__email_verifications') }}
),
bookings AS (
SELECT * FROM {{ ref('fct_exchange__bookings') }}
),
-- LOGIC
remove_supplier_company AS (
SELECT
*
FROM companies
WHERE business_type NOT IN ('1','3')
),
clean_company_transaction_orders AS (
SELECT
transaction_orders.*
FROM transaction_orders
-- Remove supplier (If any) 16/3/2023 Currently do not have any supplier in transaction orders
INNER JOIN remove_supplier_company
ON (transaction_orders.company_id = remove_supplier_company.id)
),
clean_company_bookings AS (
SELECT
bookings.*
FROM bookings
-- Remove supplier (If any) 16/3/2023 Currently do not have any supplier in transaction orders
INNER JOIN remove_supplier_company
ON (bookings.company_id = remove_supplier_company.id)
),
company_first_time_order_date AS (
SELECT
id,
company_id,
created_at
FROM clean_company_transaction_orders
WHERE
is_first_time_company_completed = '1'
),
--start defind funnel
repeat_completed_order_after_60days_activities AS (
SELECT
clean_company_transaction_orders.company_id,
0 AS is_data_cleansing_generate_row,
'repeat_completed_order_after_60days' AS activity,
MIN(clean_company_transaction_orders.created_at) AS activity_datetime
FROM clean_company_transaction_orders
LEFT JOIN company_first_time_order_date
ON (clean_company_transaction_orders.company_id = company_first_time_order_date.company_id)
WHERE
DATEDIFF(day, company_first_time_order_date.created_at, clean_company_transaction_orders.created_at) >= 60
GROUP BY
clean_company_transaction_orders.company_id
),
repeat_completed_order_after_30days_activities AS (
SELECT
clean_company_transaction_orders.company_id,
0 AS is_data_cleansing_generate_row,
'repeat_completed_order_after_30days' AS activity,
MIN(clean_company_transaction_orders.created_at) AS activity_datetime
FROM clean_company_transaction_orders
LEFT JOIN company_first_time_order_date
ON (clean_company_transaction_orders.company_id = company_first_time_order_date.company_id)
WHERE
DATEDIFF(day, company_first_time_order_date.created_at, clean_company_transaction_orders.created_at) >= 30
GROUP BY
clean_company_transaction_orders.company_id
),
first_time_order_completed_activities AS (
SELECT
clean_company_transaction_orders.company_id,
0 AS is_data_cleansing_generate_row,
'first_time_order_completed' AS activity,
MIN(clean_company_transaction_orders.created_at) AS activity_datetime
FROM clean_company_transaction_orders
WHERE
clean_company_transaction_orders.is_first_time_company_completed = 1
GROUP BY
clean_company_transaction_orders.company_id
),
first_time_order_activities AS (
SELECT
clean_company_transaction_orders.company_id,
0 AS is_data_cleansing_generate_row,
'first_time_order' AS activity,
MIN(clean_company_transaction_orders.created_at) AS activity_datetime
FROM clean_company_transaction_orders
WHERE
clean_company_transaction_orders.is_first_time_company = 1
GROUP BY
clean_company_transaction_orders.company_id
),
first_time_booking_activities AS (
SELECT
IFNULL( unclean_first_time_booking_activities.company_id , first_time_order_activities.company_id ) AS company_id,
IFF( unclean_first_time_booking_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row,
'first_time_booking' AS activity,
IFNULL( unclean_first_time_booking_activities.activity_datetime , first_time_order_activities.activity_datetime ) AS activity_datetime
FROM
(
SELECT
clean_company_bookings.company_id,
'first_time_booking' AS activity,
MIN(clean_company_bookings.booking_created_datetime) AS activity_datetime
FROM clean_company_bookings
WHERE
clean_company_bookings.is_first_time_company = 1
GROUP BY
clean_company_bookings.company_id
) AS unclean_first_time_booking_activities
FULL OUTER JOIN first_time_order_activities
ON (unclean_first_time_booking_activities.company_id = first_time_order_activities.company_id)
),
identity_documents_verified_activities AS (
SELECT
IFNULL( unclean_identity_documents_verified_activities.company_id , first_time_booking_activities.company_id ) AS company_id,
IFF( unclean_identity_documents_verified_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row,
'identity_documents_verified' AS activity,
IFNULL( unclean_identity_documents_verified_activities.activity_datetime , first_time_booking_activities.activity_datetime ) AS activity_datetime
FROM
(
SELECT
company_id,
'identity_documents_verified' AS activity,
MIN(identity_created_datetime) AS activity_datetime
FROM company_identity_documents
WHERE
identity_status = '2'
GROUP BY
company_id
) AS unclean_identity_documents_verified_activities
FULL OUTER JOIN first_time_booking_activities
ON (unclean_identity_documents_verified_activities.company_id = first_time_booking_activities.company_id)
),
identity_documents_uploaded_activities AS (
SELECT
IFNULL( unclean_identity_documents_uploaded_activities.company_id , identity_documents_verified_activities.company_id ) AS company_id,
IFF( unclean_identity_documents_uploaded_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row,
'identity_documents_uploaded' AS activity,
IFNULL( unclean_identity_documents_uploaded_activities.activity_datetime , identity_documents_verified_activities.activity_datetime ) AS activity_datetime
FROM
(
SELECT
company_id,
'identity_documents_uploaded' AS activity,
MIN(identity_created_datetime) AS activity_datetime
FROM company_identity_documents
GROUP BY
company_id
) AS unclean_identity_documents_uploaded_activities
FULL OUTER JOIN identity_documents_verified_activities
ON (unclean_identity_documents_uploaded_activities.company_id = identity_documents_verified_activities.company_id)
),
email_verified_activities AS (
SELECT
IFNULL( unclean_email_verified_activities.company_id , identity_documents_uploaded_activities.company_id ) AS company_id,
IFF( unclean_email_verified_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row,
'email_verified' AS activity,
IFNULL( unclean_email_verified_activities.activity_datetime , identity_documents_uploaded_activities.activity_datetime ) AS activity_datetime
FROM
(
SELECT
company_id,
'email_verified' AS activity,
MIN(verification_email_created_datetime) AS activity_datetime
FROM email_verifications
WHERE
is_completed = '1'
GROUP BY
company_id
) AS unclean_email_verified_activities
FULL OUTER JOIN identity_documents_uploaded_activities
ON (unclean_email_verified_activities.company_id = identity_documents_uploaded_activities.company_id)
),
register_activities AS (
SELECT
IFNULL( unclean_register_activities.company_id , email_verified_activities.company_id ) AS company_id,
IFF( unclean_register_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row,
'register' AS activity,
IFNULL( unclean_register_activities.activity_datetime , email_verified_activities.activity_datetime ) AS activity_datetime
FROM
(
SELECT
id AS company_id,
'register' AS activity,
MIN(created_at) AS activity_datetime
FROM remove_supplier_company
GROUP BY
id
) AS unclean_register_activities
FULL OUTER JOIN email_verified_activities
ON (unclean_register_activities.company_id = email_verified_activities.company_id)
),
--all activites Union into a table
union_all_activities AS (
SELECT *
FROM register_activities
{% for activity in funnel_activity[1:] %}
UNION
SELECT *
FROM {{activity}}_activities
{% endfor %}
),
union_all_activities_flag_migrated_company AS (
SELECT
union_all_activities.*,
remove_supplier_company.is_migrated_company
FROM union_all_activities
LEFT JOIN remove_supplier_company
ON (union_all_activities.company_id = remove_supplier_company.id)
),
--FINAL
sem_exchange__company_funnels AS (
SELECT
company_id,
is_data_cleansing_generate_row,
is_migrated_company,
activity_datetime,
{% for activity in funnel_activity %}
count(iff(activity = '{{activity}}', company_id, NULL))
OVER (ORDER BY activity_datetime) AS cumulative_{{activity}}_activity,
{% endfor %}
{% for activity in funnel_activity %}
count(iff(activity = '{{activity}}' AND is_migrated_company = 0, company_id, NULL))
OVER (ORDER BY activity_datetime) AS exclude_migrated_cumulative_{{activity}}_activity,
{% endfor %}
'__macro_end' AS __macro_end
FROM union_all_activities_flag_migrated_company
ORDER BY activity_datetime
)
SELECT * FROM sem_exchange__company_funnels
@@ -1,299 +0,0 @@
-- HOW TO USE IT?
-- MUST: company_migration_segment filter need to be choose
-- IMPORT
WITH companies AS (
SELECT * FROM {{ ref('dim_exchange__companies') }}
),
transaction_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
company_identity_documents AS (
SELECT * FROM {{ ref('fct_exchange__company_identity_documents') }}
),
email_verifications AS (
SELECT * FROM {{ ref('fct_exchange__email_verifications') }}
),
-- LOGIC
remove_supplier_company AS (
SELECT
*
FROM companies
WHERE business_type = 2
),
company_register_events AS (
SELECT
remove_supplier_company.id AS company_id,
'Register' AS register_event,
MIN(remove_supplier_company.created_at) AS event_datetime
FROM remove_supplier_company
GROUP BY
remove_supplier_company.id
),
company_email_verification_send_events AS (
SELECT
company_id,
'Email Verification Send' AS email_verification_send_event,
verification_email_created_datetime AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY verification_email_created_datetime) AS first_row_number
FROM email_verifications
QUALIFY
first_row_number = 1
),
company_email_verification_complete_events AS (
SELECT
company_id,
'Email Verification Complete' AS email_verification_complete_event,
verification_email_updated_datetime AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY verification_email_updated_datetime) AS first_row_number
FROM email_verifications
WHERE
is_completed = '1'
QUALIFY
first_row_number = 1
),
company_upload_identity_events AS (
SELECT
company_id,
'Upload Identity' AS upload_identity_event,
identity_created_datetime AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY identity_created_datetime) AS first_row_number
FROM company_identity_documents
QUALIFY
first_row_number = 1
),
company_approve_identity_events AS (
SELECT
company_id,
'Approve Identity' AS approve_identity_event,
identity_approved_datetine AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY identity_created_datetime) AS first_row_number
FROM company_identity_documents
WHERE
identity_status in ('2','3')
QUALIFY
first_row_number = 1
),
company_first_time_order_events AS (
SELECT
first_time_transaction_orders.company_id,
companies.id,
'First Time Order' AS first_time_order_event,
first_time_transaction_orders.created_at AS event_datetime
FROM
(SELECT
*,
row_number() OVER
(PARTITION BY transaction_orders.company_id
ORDER BY transaction_orders.created_at) AS row_rank_index
FROM transaction_orders
QUALIFY
row_rank_index = 1
) AS first_time_transaction_orders
INNER JOIN companies
ON(first_time_transaction_orders.company_id = companies.id)
),
company_funnel_events AS (
SELECT
company_id,
register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_register_events
WHERE register_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_email_verification_send_events
WHERE email_verification_send_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_email_verification_complete_events
WHERE email_verification_complete_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_upload_identity_events
WHERE upload_identity_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_approve_identity_events
WHERE approve_identity_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
first_time_order_event,
event_datetime
FROM company_first_time_order_events
WHERE first_time_order_event IS NOT NULL
),
--FINAL
sem_exchange__company_funnel_events AS (
SELECT
'all_company' AS company_migration_segment,
company_funnel_events.event_datetime,
company_funnel_events.company_id,
remove_supplier_company.is_migrated_company,
count(iff(company_funnel_events.register_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS register_company,
count(iff(company_funnel_events.email_verification_send_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_send_company,
count(iff(company_funnel_events.email_verification_complete_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_complete_company,
count(iff(company_funnel_events.upload_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS upload_identity_company,
count(iff(company_funnel_events.approve_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS approve_identity_event,
count(iff(company_funnel_events.first_time_order_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS first_time_order_company
FROM company_funnel_events
INNER JOIN remove_supplier_company
ON (company_funnel_events.company_id = remove_supplier_company.id)
UNION
SELECT
'exclude_migrated_company' AS company_migration_segment,
company_funnel_events.event_datetime,
company_funnel_events.company_id,
remove_supplier_company.is_migrated_company,
count(iff(company_funnel_events.register_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS register_company,
count(iff(company_funnel_events.email_verification_send_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_send_company,
count(iff(company_funnel_events.email_verification_complete_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_complete_company,
count(iff(company_funnel_events.upload_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS upload_identity_company,
count(iff(company_funnel_events.approve_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS approve_identity_event,
count(iff(company_funnel_events.first_time_order_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS first_time_order_company
FROM company_funnel_events
INNER JOIN remove_supplier_company
ON (company_funnel_events.company_id = remove_supplier_company.id)
WHERE remove_supplier_company.is_migrated_company = '0'
)
SELECT count(*) FROM sem_exchange__company_funnel_events
-84
View File
@@ -1,84 +0,0 @@
WITH container_packing_lists AS (
SELECT * FROM {{ ref('base_shipping__container_packing_lists') }}
),
containers AS (
SELECT * FROM {{ ref('base_shipping__containers') }}
),
packing_lists AS (
SELECT * FROM {{ ref('base_shipping__packing_lists') }}
),
packages AS (
SELECT * FROM {{ ref('base_shipping__packages') }}
),
company_modules AS (
SELECT * FROM {{ ref('base_shipping__company_modules') }}
),
company_connections AS (
SELECT * FROM {{ ref('base_shipping__company_connections') }}
),
orders AS (
SELECT * FROM {{ ref('base_shipping__orders') }}
)
-- LOGIC
company_module_join_companies AS (
SELECT
company_modules.id AS company_id,
company_modules.name AS company_name,
company_connections.invitee_reference AS marking
FROM
company_modules
LEFT JOIN company_connections
ON (company_modules.id = company_connections.invitee_id)
),
final AS (
SELECT
container_packing_lists.*,
containers.*,
packing_lists.*,
packages.*,
packages.created_at AS packages_created_at,
((packages.width/100) * (packages.height/100) * (packages.length/100) * packages.quantity) as cbm
FROM container_packing_lists
JOIN containers
ON (container_packing_lists.container_id = containers.id)
JOIN packing_lists
ON (container_packing_lists.packing_list_id = packing_lists.id)
JOIN packages
ON (packages.packing_list_id = packing_lists.id and packages.deleted_at is null)
WHERE packing_lists.deleted_at is null AND packages_created_at >= '2022-01-01')
-- ),
-- final_group_by_company_module AS (
-- SELECT
-- final.owner_id,
-- SUM(final.cbm) AS life_time_cbm
-- FROM final
-- GROUP BY
-- owner_id
-- )
SELECT * FROM final
-- SELECT
-- containers.*, packing_lists.id as packing_list_id, SUM((packages.width/100) * (packages.height/100) * (packages.length/100) * packages.quantity) as cbm
-- FROM
-- container_packing_lists as packed_containers
-- JOIN containers ON packed_containers.container_id = containers.id
-- JOIN packing_lists ON packed_containers.packing_list_id = packing_lists.id
-- JOIN packages ON (packages.packing_list_id = packing_lists.id and packages.deleted_at is null)
-- WHERE packing_lists.deleted_at is null GROUP BY packing_lists.id;