From 22723bdd056d6e874f325539de0249ab1f25a49e Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Wed, 2 Aug 2023 03:54:13 +0000 Subject: [PATCH 1/5] Sales activation model --- ...oglesheet__sales_team_lead_activations.sql | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 models/staging/googlesheet/stg_googlesheet__sales_team_lead_activations.sql diff --git a/models/staging/googlesheet/stg_googlesheet__sales_team_lead_activations.sql b/models/staging/googlesheet/stg_googlesheet__sales_team_lead_activations.sql new file mode 100644 index 0000000..ffd1f37 --- /dev/null +++ b/models/staging/googlesheet/stg_googlesheet__sales_team_lead_activations.sql @@ -0,0 +1,78 @@ +-- IMPORT +WITH lead_activations AS ( + SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_leads_activations') }} +), + +-- LOGIC +parse_json_leads AS ( + SELECT + _airbyte_ab_id AS sales_team_lead_activation_id, + parse_json(_airbyte_data) AS leads_json, + _airbyte_emitted_at AS _airbyte_emitted_datetime + + FROM lead_activations +), + +flatten_json AS ( + + SELECT + sales_team_lead_activation_id, + leads_json['agent_name']::STRING AS agent_name, + leads_json['company_marking']::STRING AS company_marking_id, + leads_json['contact_number']::STRING AS contact_number, + leads_json['email']::STRING AS contact_email, + leads_json['sorce']::STRING AS contact_method, + leads_json['tag']::STRING AS contact_status, + leads_json['user_name']::STRING AS contact_name, + leads_json['remark']::STRING AS sales_team_remark, + COALESCE( + TRY_TO_TIMESTAMP(leads_json['contacted_datetime']::STRING, 'DD-MM-YYYY'), + TRY_TO_TIMESTAMP(leads_json['contacted_datetime']::STRING, 'DD-MM-YYYY HH24:MI:SS') + ) AS sales_team_contacted_datetime, + COALESCE( + TRY_TO_TIMESTAMP(leads_json['register_date']::STRING, 'DD-MM-YYYY'), + TRY_TO_TIMESTAMP(leads_json['register_date']::STRING, 'DD-MM-YYYY HH24:MI:SS') + ) AS exchange_registered_datetime, + COALESCE( + TRY_TO_TIMESTAMP(leads_json['user_first_reply_datetime']::STRING, 'DD-MM-YYYY'), + TRY_TO_TIMESTAMP(leads_json['user_first_reply_datetime']::STRING, 'DD-MM-YYYY HH24:MI:SS') + ) AS first_reply_datetime, + + _airbyte_emitted_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM parse_json_leads +), + +-- FINAL +final__stg_googlesheet__sales_team_lead_activations AS ( + + SELECT + -- ids + sales_team_lead_activation_id, + + -- dimensions + company_marking_id, + agent_name, + contact_name, + contact_number, + contact_email, + contact_method, + contact_status, + sales_team_remark, + + -- measures + + -- date/times + sales_team_contacted_datetime, + first_reply_datetime, + exchange_registered_datetime, + + -- metadata + _airbyte_emitted_datetime, + _dbt_ran_datetime + + FROM flatten_json +) + +SELECT * FROM final__stg_googlesheet__sales_team_lead_activations \ No newline at end of file From 973f835d15cf41bdded6f88c5594066d919debfa Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Wed, 2 Aug 2023 04:12:14 +0000 Subject: [PATCH 2/5] Change name --- ...oglesheet__sales_team_lead_activations.sql | 78 ------------------- 1 file changed, 78 deletions(-) delete mode 100644 models/staging/googlesheet/stg_googlesheet__sales_team_lead_activations.sql diff --git a/models/staging/googlesheet/stg_googlesheet__sales_team_lead_activations.sql b/models/staging/googlesheet/stg_googlesheet__sales_team_lead_activations.sql deleted file mode 100644 index ffd1f37..0000000 --- a/models/staging/googlesheet/stg_googlesheet__sales_team_lead_activations.sql +++ /dev/null @@ -1,78 +0,0 @@ --- IMPORT -WITH lead_activations AS ( - SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_leads_activations') }} -), - --- LOGIC -parse_json_leads AS ( - SELECT - _airbyte_ab_id AS sales_team_lead_activation_id, - parse_json(_airbyte_data) AS leads_json, - _airbyte_emitted_at AS _airbyte_emitted_datetime - - FROM lead_activations -), - -flatten_json AS ( - - SELECT - sales_team_lead_activation_id, - leads_json['agent_name']::STRING AS agent_name, - leads_json['company_marking']::STRING AS company_marking_id, - leads_json['contact_number']::STRING AS contact_number, - leads_json['email']::STRING AS contact_email, - leads_json['sorce']::STRING AS contact_method, - leads_json['tag']::STRING AS contact_status, - leads_json['user_name']::STRING AS contact_name, - leads_json['remark']::STRING AS sales_team_remark, - COALESCE( - TRY_TO_TIMESTAMP(leads_json['contacted_datetime']::STRING, 'DD-MM-YYYY'), - TRY_TO_TIMESTAMP(leads_json['contacted_datetime']::STRING, 'DD-MM-YYYY HH24:MI:SS') - ) AS sales_team_contacted_datetime, - COALESCE( - TRY_TO_TIMESTAMP(leads_json['register_date']::STRING, 'DD-MM-YYYY'), - TRY_TO_TIMESTAMP(leads_json['register_date']::STRING, 'DD-MM-YYYY HH24:MI:SS') - ) AS exchange_registered_datetime, - COALESCE( - TRY_TO_TIMESTAMP(leads_json['user_first_reply_datetime']::STRING, 'DD-MM-YYYY'), - TRY_TO_TIMESTAMP(leads_json['user_first_reply_datetime']::STRING, 'DD-MM-YYYY HH24:MI:SS') - ) AS first_reply_datetime, - - _airbyte_emitted_datetime, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - - FROM parse_json_leads -), - --- FINAL -final__stg_googlesheet__sales_team_lead_activations AS ( - - SELECT - -- ids - sales_team_lead_activation_id, - - -- dimensions - company_marking_id, - agent_name, - contact_name, - contact_number, - contact_email, - contact_method, - contact_status, - sales_team_remark, - - -- measures - - -- date/times - sales_team_contacted_datetime, - first_reply_datetime, - exchange_registered_datetime, - - -- metadata - _airbyte_emitted_datetime, - _dbt_ran_datetime - - FROM flatten_json -) - -SELECT * FROM final__stg_googlesheet__sales_team_lead_activations \ No newline at end of file From 851c9afbb935718f3bf5a9985f7c38fef4eddb07 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Wed, 2 Aug 2023 04:12:27 +0000 Subject: [PATCH 3/5] sales exchange lead activations --- ...sheet__sales_exchange_lead_activations.sql | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 models/staging/googlesheet/stg_googlesheet__sales_exchange_lead_activations.sql diff --git a/models/staging/googlesheet/stg_googlesheet__sales_exchange_lead_activations.sql b/models/staging/googlesheet/stg_googlesheet__sales_exchange_lead_activations.sql new file mode 100644 index 0000000..917d47f --- /dev/null +++ b/models/staging/googlesheet/stg_googlesheet__sales_exchange_lead_activations.sql @@ -0,0 +1,78 @@ +-- IMPORT +WITH lead_activations AS ( + SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_leads_activations') }} +), + +-- LOGIC +parse_json_leads AS ( + SELECT + _airbyte_ab_id AS sales_team_lead_activation_id, + parse_json(_airbyte_data) AS leads_json, + _airbyte_emitted_at AS _airbyte_emitted_datetime + + FROM lead_activations +), + +flatten_json AS ( + + SELECT + sales_team_lead_activation_id, + leads_json['agent_name']::STRING AS agent_name, + leads_json['company_marking']::STRING AS company_marking_id, + leads_json['contact_number']::STRING AS contact_number, + leads_json['email']::STRING AS contact_email, + leads_json['sorce']::STRING AS contact_method, + leads_json['tag']::STRING AS contact_status, + leads_json['user_name']::STRING AS contact_name, + leads_json['remark']::STRING AS sales_team_remark, + COALESCE( + TRY_TO_TIMESTAMP(leads_json['contacted_datetime']::STRING, 'DD-MM-YYYY'), + TRY_TO_TIMESTAMP(leads_json['contacted_datetime']::STRING, 'DD-MM-YYYY HH24:MI:SS') + ) AS sales_team_contacted_datetime, + COALESCE( + TRY_TO_TIMESTAMP(leads_json['register_date']::STRING, 'DD-MM-YYYY'), + TRY_TO_TIMESTAMP(leads_json['register_date']::STRING, 'DD-MM-YYYY HH24:MI:SS') + ) AS exchange_registered_datetime, + COALESCE( + TRY_TO_TIMESTAMP(leads_json['user_first_reply_datetime']::STRING, 'DD-MM-YYYY'), + TRY_TO_TIMESTAMP(leads_json['user_first_reply_datetime']::STRING, 'DD-MM-YYYY HH24:MI:SS') + ) AS first_reply_datetime, + + _airbyte_emitted_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM parse_json_leads +), + +-- FINAL +final__stg_googlesheet__sales_exchange_lead_activations AS ( + + SELECT + -- ids + sales_team_lead_activation_id, + + -- dimensions + company_marking_id, + agent_name, + contact_name, + contact_number, + contact_email, + contact_method, + contact_status, + sales_team_remark, + + -- measures + + -- date/times + sales_team_contacted_datetime, + first_reply_datetime, + exchange_registered_datetime, + + -- metadata + _airbyte_emitted_datetime, + _dbt_ran_datetime + + FROM flatten_json +) + +SELECT * FROM final__stg_googlesheet__sales_exchange_lead_activations \ No newline at end of file From 4eb48eb07000832cc2f1c4145e6341bf27e9fba3 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Wed, 2 Aug 2023 04:12:39 +0000 Subject: [PATCH 4/5] sales exchange lead activations --- .../rep_exchange__sales_lead_conversions.sql | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 models/marts/reporting/rep_exchange__sales_lead_conversions.sql diff --git a/models/marts/reporting/rep_exchange__sales_lead_conversions.sql b/models/marts/reporting/rep_exchange__sales_lead_conversions.sql new file mode 100644 index 0000000..b189f0a --- /dev/null +++ b/models/marts/reporting/rep_exchange__sales_lead_conversions.sql @@ -0,0 +1,290 @@ +-- IMPORTS +WITH leads AS ( + SELECT * FROM {{ ref('fct_googlesheet__sales_exchange_lead_activations') }} +), + +companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + +bookings AS ( + SELECT * FROM {{ ref('fct_exchange__bookings') }} +), + +transactions AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} +), + +-- LOGIC +join_booking_transaction AS ( + + SELECT + leads.sales_team_lead_activation_id, + leads.company_marking_id, + leads.agent_name, + leads.contact_name, + leads.contact_number, + leads.contact_email, + leads.contact_method, + leads.contact_status, + leads.sales_team_contacted_datetime, + leads.first_reply_datetime, + leads.exchange_registered_datetime, + + companies.company_id, + + bookings.booking_id, + bookings.booking_marking_id, + bookings.user_id, + bookings.service_type, + bookings.status AS booking_status, + bookings.is_first_time_company, + bookings.is_first_time_company_completed, + bookings.is_first_time_user, + bookings.is_first_time_user_completed, + bookings.booking_created_datetime, + bookings.booking_approved_datetime, + bookings.booking_completed_datetime, + + transactions.transaction_order_id, + transactions.transaction_type, + transactions.payment_method, + transactions.total_value_rm, + transactions.order_status AS transaction_order_status, + transactions.order_created_datetime AS transaction_created_datetime, + transactions.order_approved_datetime AS transaction_approved_datetime, + transactions.order_completed_datetime AS transaction_completed_datetime + + FROM leads + + LEFT JOIN companies + ON (leads.company_marking_id = companies.company_marking_id) + + LEFT JOIN bookings + ON (companies.company_id = bookings.company_id) + + LEFT JOIN transactions + ON (bookings.booking_id = transactions.booking_id) + +), + +data_enrich_grouped AS ( + + SELECT + company_id, + sales_team_lead_activation_id, + company_marking_id, + agent_name, + contact_name, + contact_number, + contact_email, + contact_method, + contact_status, + exchange_registered_datetime, + first_reply_datetime, + + MIN(sales_team_contacted_datetime) AS sales_team_first_contacted_datetime, + + -- Count of orders placed after contacted by sales team + SUM( + CASE + WHEN transaction_order_status = 'COMPLETED' AND contact_status = 'Contacted' AND transaction_created_datetime > sales_team_contacted_datetime + THEN 1 + ELSE 0 + END + ) AS count_transacted_after_contact, + + -- Number of days that a company is first contacted after its registration date in exchange + MIN(TIMEDIFF(hour, exchange_registered_datetime, sales_team_contacted_datetime)) / 24.0 AS days_between_registration_and_first_contacted, + + -- Boolean - Bookings and orders made BEFORE contacted datetime + CASE + WHEN MIN( booking_created_datetime ) < sales_team_contacted_datetime THEN '1' + ELSE '0' + END AS has_booked_before_contacted, + CASE + WHEN MIN( transaction_created_datetime ) < sales_team_contacted_datetime THEN '1' + ELSE '0' + END AS has_ordered_before_contacted, + + -- Boolean - Bookings and orders made AFTER contacted datetime and successful contact + CASE + WHEN MAX( booking_created_datetime ) > sales_team_contacted_datetime AND contact_status = 'Contacted' THEN '1' + ELSE '0' + END AS has_booked_after_contacted, + CASE + WHEN MAX( transaction_created_datetime ) > sales_team_contacted_datetime AND contact_status = 'Contacted' THEN '1' + ELSE '0' + END AS has_ordered_after_contacted, + + -- Hours - Booking and orders made BEFORE contacted datetime + MIN( + CASE + WHEN booking_created_datetime < sales_team_contacted_datetime + THEN TIMEDIFF(minute, booking_created_datetime, sales_team_contacted_datetime) / 60 + END + ) AS hours_last_booked_before_contacted, + MIN( + CASE + WHEN transaction_created_datetime < sales_team_contacted_datetime + AND transaction_order_status NOT IN ('SUSPENDED', 'REJECTED') + THEN TIMEDIFF(minute, transaction_created_datetime, sales_team_contacted_datetime) / 60 + END + ) AS hours_last_ordered_before_contacted, + + -- Hours - Booking and orders made AFTER contacted datetime and successful contact + MIN( + CASE + WHEN ( sales_team_contacted_datetime < booking_created_datetime ) AND contact_status = 'Contacted' + THEN TIMEDIFF(minute, sales_team_contacted_datetime, booking_created_datetime) / 60 + END + ) AS hours_first_booked_after_contacted, + MIN( + CASE + WHEN ( sales_team_contacted_datetime < transaction_created_datetime ) AND contact_status = 'Contacted' + AND transaction_order_status NOT IN ('SUSPENDED', 'REJECTED') + THEN TIMEDIFF(minute, sales_team_contacted_datetime, transaction_created_datetime) / 60 + END + ) AS hours_first_ordered_after_contacted, + + -- Hours -- Time taken for customer to reply after first contact + CASE + WHEN contact_status = 'Contacted' THEN TIMEDIFF(minute, sales_team_contacted_datetime, first_reply_datetime) / 60 + END AS hours_first_reply, + + -- total order value BEFORE contacted datetime + SUM( + CASE WHEN sales_team_contacted_datetime > transaction_created_datetime THEN total_value_rm ELSE 0 END + ) AS total_value_rm_before_contact, + SUM( + CASE WHEN transaction_order_status IN ('APPROVED', 'COMPLETED') + AND sales_team_contacted_datetime > transaction_created_datetime THEN total_value_rm ELSE 0 END + ) AS total_value_rm_approved_completed_before_contact, + + -- total order value AFTER contacted datetime + SUM( + CASE WHEN sales_team_contacted_datetime < transaction_created_datetime + AND contact_status = 'Contacted' THEN total_value_rm ELSE 0 END + ) AS total_value_rm_after_contact, + SUM( + CASE WHEN transaction_order_status IN ('APPROVED', 'COMPLETED') + AND sales_team_contacted_datetime < transaction_created_datetime AND contact_status = 'Contacted' THEN total_value_rm ELSE 0 END + ) AS total_value_rm_approved_completed_after_contact, + + -- Leads categorization + CASE + -- Category 1: Fail to contact customer (status = 'Failed contacted') + -- This includes customers who do not reply and customers who rejected our offer + WHEN + ( contact_status != 'Contacted' AND has_booked_before_contacted = 0 AND has_ordered_before_contacted = 0 + AND has_booked_after_contacted = 0 AND has_ordered_after_contacted = 0 ) OR + ( contact_status != 'Contacted' AND has_booked_before_contacted = 1 AND has_ordered_before_contacted = 0 + AND has_booked_after_contacted = 0 AND has_ordered_after_contacted = 0 ) OR + ( contact_status != 'Contacted' AND has_booked_before_contacted = 1 AND has_ordered_before_contacted = 1 + AND has_booked_after_contacted = 0 AND has_ordered_after_contacted = 0 ) + THEN 'Unreachable' + + -- Category 2: Existing customers + -- Only successfully contacted customers are included + WHEN + ( contact_status = 'Contacted' AND has_booked_before_contacted = 1 AND has_ordered_before_contacted = 1 + AND has_booked_after_contacted = 0 AND has_ordered_after_contacted = 0 ) OR + ( contact_status = 'Contacted' AND has_booked_before_contacted = 1 AND has_ordered_before_contacted = 1 + AND has_booked_after_contacted = 1 AND has_ordered_after_contacted = 0 ) OR + ( contact_status = 'Contacted' AND has_booked_before_contacted = 1 AND has_ordered_before_contacted = 1 + AND has_booked_after_contacted = 1 AND has_ordered_after_contacted = 1 ) + THEN 'Existing Customer' + + -- Category 3: No action from contacted customers + WHEN + ( contact_status = 'Contacted' AND has_booked_before_contacted = 0 AND has_ordered_before_contacted = 0 + AND has_booked_after_contacted = 0 AND has_ordered_after_contacted = 0 ) OR + ( contact_status = 'Contacted' AND has_booked_before_contacted = 1 AND has_ordered_before_contacted = 0 + AND has_booked_after_contacted = 0 AND has_ordered_after_contacted = 0 ) + THEN 'Unengaged' + + -- Category 4: Leads placed booking but not order. + -- Only successfully contacted customers are included + WHEN + ( contact_status = 'Contacted' AND has_booked_before_contacted = 0 AND has_ordered_before_contacted = 0 + AND has_booked_after_contacted = 1 AND has_ordered_after_contacted = 0 ) OR + ( contact_status = 'Contacted' AND has_booked_before_contacted = 1 AND has_ordered_before_contacted = 0 + AND has_booked_after_contacted = 1 AND has_ordered_after_contacted = 0 ) + THEN 'Potential Opportunity' + + -- Category 5: Successfully converted a lead + WHEN + ( contact_status = 'Contacted' AND has_booked_before_contacted = 0 AND has_ordered_before_contacted = 0 + AND has_booked_after_contacted = 1 AND has_ordered_after_contacted = 1 ) OR + ( contact_status = 'Contacted' AND has_booked_before_contacted = 1 AND has_ordered_before_contacted = 0 + AND has_booked_after_contacted = 1 AND has_ordered_after_contacted = 1 ) + THEN 'Converted' + + END AS leads_categorization, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM join_booking_transaction + + GROUP BY + company_id, + sales_team_lead_activation_id, + company_marking_id, + agent_name, + contact_name, + contact_number, + contact_email, + contact_method, + contact_status, + sales_team_contacted_datetime, + first_reply_datetime, + exchange_registered_datetime +), + +-- FINAL +final__rep_exchange__sales_lead_conversions AS ( + + SELECT + --ids + company_id, + sales_team_lead_activation_id, + company_marking_id, + + --dimensions + agent_name, + contact_method, + contact_name, + contact_number, + contact_email, + contact_status, + has_booked_before_contacted, + has_ordered_before_contacted, + has_booked_after_contacted, + has_ordered_after_contacted, + leads_categorization, + + --measures + days_between_registration_and_first_contacted, + hours_last_booked_before_contacted, + hours_last_ordered_before_contacted, + hours_first_booked_after_contacted, + hours_first_ordered_after_contacted, + hours_first_reply, + count_transacted_after_contact, + total_value_rm_before_contact, + total_value_rm_approved_completed_before_contact, + total_value_rm_after_contact, + total_value_rm_approved_completed_after_contact, + + --datetime + sales_team_first_contacted_datetime, + first_reply_datetime, + exchange_registered_datetime, + + --metadata + _dbt_ran_datetime + + FROM data_enrich_grouped +) + +SELECT * FROM final__rep_exchange__sales_lead_conversions \ No newline at end of file From da82a4f1903e8bb1428aa516e10d06a340cc245a Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Wed, 2 Aug 2023 04:12:49 +0000 Subject: [PATCH 5/5] sales exchange lead activations --- ...sheet__sales_exchange_lead_activations.sql | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 models/marts/warehouse/fct_googlesheet__sales_exchange_lead_activations.sql diff --git a/models/marts/warehouse/fct_googlesheet__sales_exchange_lead_activations.sql b/models/marts/warehouse/fct_googlesheet__sales_exchange_lead_activations.sql new file mode 100644 index 0000000..d6ec6c6 --- /dev/null +++ b/models/marts/warehouse/fct_googlesheet__sales_exchange_lead_activations.sql @@ -0,0 +1,33 @@ +-- IMPORT +WITH lead_activations AS ( + SELECT * FROM {{ ref('stg_googlesheet__sales_exchange_lead_activations') }} +), + + +-- FINAL +final__fct_googlesheet__sales_exchange_lead_activations AS ( + SELECT + sales_team_lead_activation_id, + company_marking_id, + agent_name, + contact_name, + contact_number, + contact_email, + contact_method, + contact_status, + + -- measures + + -- date/times + sales_team_contacted_datetime, + first_reply_datetime, + exchange_registered_datetime, + + -- metadata + sales_team_remark, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM lead_activations +) + +SELECT * FROM final__fct_googlesheet__sales_exchange_lead_activations \ No newline at end of file