From 77f4144c74d5166cef28996343a3599e219bbe9f Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Thu, 13 Jul 2023 15:25:46 +0000 Subject: [PATCH 01/11] Change miss spelling --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3bead35..ca33c80 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Short naming convention lists that need to follow - Use names based on the _business_ terminology, rather than the source terminology. -- Each model should have a primary key that can identify the unique row, and should be named `_id`, e.g. `account_id` – this makes it easier to know what `id` is being referenced in downstream joined models. +- Each model should have a primary key that can identify the unique row, and should be named `_id`, e.g. `account_id` - this makes it easier to know what `id` is being referenced in downstream joined models. - If a surrogate key is created, it should be named `_sk`. From c9e81d9958d304b5089881cdc30a4310d6a190fc Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Thu, 13 Jul 2023 15:26:52 +0000 Subject: [PATCH 02/11] Debug --- models/marts/reporting/rep_exchange__company_lists.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/reporting/rep_exchange__company_lists.sql b/models/marts/reporting/rep_exchange__company_lists.sql index af2131c..793ac3d 100644 --- a/models/marts/reporting/rep_exchange__company_lists.sql +++ b/models/marts/reporting/rep_exchange__company_lists.sql @@ -21,7 +21,7 @@ company_agg AS ( companies.company_marking_id, companies.company_created_datetime, COUNT(transaction_orders.transaction_order_id) AS count_order, - COUNT(IFF(transaction_orders.order_status = 'completed', transaction_orders.transaction_order_id, null)) AS count_completed_order, + COUNT(IFF(transaction_orders.order_status = 'COMPLETED', transaction_orders.transaction_order_id, null)) AS count_completed_order, MAX(transaction_orders.order_created_datetime) AS last_order_datetime, COALESCE(last_order_datetime, company_created_datetime) AS last_activity_datetime, IFF(companies.company_created_datetime < '2021-07-01', 1, 0) AS is_wrong_created_datetime_company, From 725d2ef24a988e14cd47901669bad76d0a2daa68 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Thu, 13 Jul 2023 15:27:56 +0000 Subject: [PATCH 03/11] googlesheet sources --- .../googlesheet/_googlesheet__sources.yml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 models/staging/googlesheet/_googlesheet__sources.yml diff --git a/models/staging/googlesheet/_googlesheet__sources.yml b/models/staging/googlesheet/_googlesheet__sources.yml new file mode 100644 index 0000000..800c970 --- /dev/null +++ b/models/staging/googlesheet/_googlesheet__sources.yml @@ -0,0 +1,50 @@ +version: 2 + +sources: + - name: src_googlesheet_airbyte + description: '{{ doc("src_googlesheet_airbyte") }}' + database: DEV_CIEF_RAW_DB + schema: GOOGLESHEET_AIRBYTE + loader: Airbyte + loaded_at_field: _airbyte_emitted_at + freshness: + warn_after: {count: 26, period: hour} + error_after: {count: 48, period: hour} + meta: + owner: "@yam" + model_maturity: prod + tags: + - googlesheet + - daily + + + tables: + - name: _airbyte_raw_leads_activations + description: This data is get from sales gsheet, sales will call user once they register & not put any order + columns: + - name: _airbyte_ab_id + description: Primary key for '_airbyte_raw_leads_activations' + tests: + - unique + - not_null + + - name: _airbyte_data + description: JSON data which come from gsheet + tests: + - not_null + + + - name: _airbyte_raw_event_and_holiday_lists + description: Data retrieved from gsheet, comprises dates for events and holidays that may or may not impact operation of company. + columns: + - name: _airbyte_ab_id + description: Primary key for '_airbyte_raw_event_and_holiday_lists' + tests: + - unique + - not_null + + - name: _airbyte_data + description: JSON data which come from gsheet + tests: + - not_null + From 3c71b9b80dabe0e71a8b3da02e32b7dfb9a6ad45 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Thu, 13 Jul 2023 15:28:09 +0000 Subject: [PATCH 04/11] googlesheet docs --- .../staging/googlesheet/_googlesheet__docs.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 models/staging/googlesheet/_googlesheet__docs.md diff --git a/models/staging/googlesheet/_googlesheet__docs.md b/models/staging/googlesheet/_googlesheet__docs.md new file mode 100644 index 0000000..c5c02ce --- /dev/null +++ b/models/staging/googlesheet/_googlesheet__docs.md @@ -0,0 +1,17 @@ +{% docs src_googlesheet_airbyte %} + +Application link: https://airbyte.com/ + +Architecture: Sales Interface Sheet -> AppScript -> Clone Centralisations Sheet -> Airbyte -> Snowflake + +Data is extract from googlesheet [gsheet_clone_centralisations](https://docs.google.com/spreadsheets/d/1GD2l-QnbLY-CfC3BCAouNCM3ROzboCKgeRXEhl64NUg/edit#gid=0), using airbyte. + +Before load into 'gsheet_clone_centralisations' it will clone the data from sales gsheet interface [sales_enq_recoarding](https://docs.google.com/spreadsheets/d/1ENYzT_cihpFiNfc7DRZpSRf-PXi_R6cmRn9XKY8a_Vk/edit#gid=376414069) + +AppScript to clone the data before airbyte clone into snowflake [AppScript](https://script.google.com/u/1/home/projects/19J_a0P4FB-DVkNGz0THleNp7ISXZxrh8eyNvKxZyoXTP8DOJMWz1VsiK) + +- `_airbyte_emitted_at` will add into the raw data by airbyte when extract, it is UTC timezone + +- Every 11pm Malaysia Timezone, AppScript will clone the data into Clone Centralisations Sheet and 12am Malaysia Timezone airbyte will clone the data into snowflake + +{% enddocs %} \ No newline at end of file From 732c0c516e60ea53bb594755e27011de62d2c550 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Thu, 13 Jul 2023 15:28:35 +0000 Subject: [PATCH 05/11] googlesheet .gitkeep --- models/staging/googlesheet/base/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 models/staging/googlesheet/base/.gitkeep diff --git a/models/staging/googlesheet/base/.gitkeep b/models/staging/googlesheet/base/.gitkeep new file mode 100644 index 0000000..e69de29 From 98d6d8da20bb84c3c9325b852c1013c1a0a09fed Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Sat, 15 Jul 2023 04:01:34 +0000 Subject: [PATCH 06/11] Exchange add source --- .../staging/exchange/_exchange__sources.yml | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/models/staging/exchange/_exchange__sources.yml b/models/staging/exchange/_exchange__sources.yml index f4e3550..20620be 100644 --- a/models/staging/exchange/_exchange__sources.yml +++ b/models/staging/exchange/_exchange__sources.yml @@ -21,7 +21,56 @@ sources: - daily - tables: + tables: + - name: account_statements + description: "" + columns: + - name: id + description: Primary key for 'account_statements' + + + - name: seasonal_segment + description: Use to assign company into temporary segment, this function normally use in campaign + columns: + - name: id + description: Primary key for 'seasonal_segment' + + + - name: statement_accounts + description: "" + columns: + - name: id + description: Primary key for 'statement_accounts' + + + - name: statement_transactions + description: "" + columns: + - name: id + description: Primary key for 'statement_transactions' + + + - name: statement_transaction_owners + description: "" + columns: + - name: id + description: Primary key for 'statement_transaction_owners' + + + - name: vouchers + description: "" + columns: + - name: id + description: Primary key for 'vouchers' + + + - name: voucher_redemptions + description: "" + columns: + - name: id + description: Primary key for 'voucher_redemptions' + + - name: activity_log description: Contains most table's activity log that CUD (created, updated, deleted) in www.exchange.cief-malaysia.com columns: From a5968114043e6d4f35b15aca2e43d4d82db3b6c6 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Mon, 17 Jul 2023 05:16:11 +0000 Subject: [PATCH 07/11] Update company_adds increamental bug --- .../base/base_exchange__company_addresses.sql | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/models/staging/exchange/base/base_exchange__company_addresses.sql b/models/staging/exchange/base/base_exchange__company_addresses.sql index a828656..de2a9ec 100644 --- a/models/staging/exchange/base/base_exchange__company_addresses.sql +++ b/models/staging/exchange/base/base_exchange__company_addresses.sql @@ -24,30 +24,6 @@ addresses_rename AS ( FROM addresses ), -remove_deleted_addresses AS ( - SELECT - * - - FROM addresses_rename - - WHERE - deleted_datetime IS NULL -), - -lastest_billing_and_non_billing_addresses AS ( - SELECT - *, - -- 1 will be latest updated address, rank by updated_at date - row_number() OVER - (PARTITION BY remove_deleted_addresses.company_id, remove_deleted_addresses.is_billing_address - ORDER BY remove_deleted_addresses.updated_datetime DESC) AS lastest_address_rank - - FROM remove_deleted_addresses - - QUALIFY - lastest_address_rank = 1 -), - -- FINAL final__base_exchange__company_addresses AS ( @@ -75,7 +51,7 @@ final__base_exchange__company_addresses AS ( -- metadata _dbt_ran_datetime - FROM lastest_billing_and_non_billing_addresses + FROM addresses_rename ) SELECT * FROM final__base_exchange__company_addresses \ No newline at end of file From 5b982e089ac50f30e1cc038c70d5fec00d0a3d1a Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Mon, 17 Jul 2023 05:16:19 +0000 Subject: [PATCH 08/11] Update company_adds increamental bug --- models/marts/warehouse/dim_exchange__companies.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/marts/warehouse/dim_exchange__companies.sql b/models/marts/warehouse/dim_exchange__companies.sql index d27d5b9..ce13260 100644 --- a/models/marts/warehouse/dim_exchange__companies.sql +++ b/models/marts/warehouse/dim_exchange__companies.sql @@ -4,7 +4,7 @@ WITH companies AS ( ), company_addresses AS ( - SELECT * FROM {{ ref('stg_exchange__company_addresses') }} + SELECT * FROM {{ ref('int_exchange__latest_company_billing_addresses') }} ), company_wallets AS ( @@ -230,4 +230,4 @@ final__dim_exchange__companies AS ( FROM companies_rename_and_join_table ) -SELECT * FROM final__dim_exchange__companies +SELECT *FROM final__dim_exchange__companies \ No newline at end of file From 26dc5271263d326a344f1ec677582fc45b894f02 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Mon, 17 Jul 2023 05:16:28 +0000 Subject: [PATCH 09/11] Update company_adds increamental bug --- ...ange__latest_company_billing_addresses.sql | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 models/intermediate/int_exchange__latest_company_billing_addresses.sql diff --git a/models/intermediate/int_exchange__latest_company_billing_addresses.sql b/models/intermediate/int_exchange__latest_company_billing_addresses.sql new file mode 100644 index 0000000..e94daff --- /dev/null +++ b/models/intermediate/int_exchange__latest_company_billing_addresses.sql @@ -0,0 +1,84 @@ +-- IMPORTS +WITH company_addresses AS ( + SELECT * FROM {{ ref('stg_exchange__company_addresses') }} +), + +-- LOGIC +remove_deleted_addresses AS ( + SELECT + * + + FROM company_addresses + + WHERE + deleted_datetime IS NULL +), + +lastest_billing_and_non_billing_addresses AS ( + SELECT + *, + -- 1 will be latest updated address, rank by updated_at date + row_number() OVER + (PARTITION BY remove_deleted_addresses.company_id, remove_deleted_addresses.is_billing_address + ORDER BY remove_deleted_addresses.updated_datetime DESC) AS lastest_address_rank + + FROM remove_deleted_addresses + + QUALIFY + lastest_address_rank = 1 +), + +-- FINAL +final__int_exchange__latest_company_billing_addresses AS ( + + SELECT + -- ids + company_address_id, + company_id, + country_id, + state_id, + district_id, + google_place_id, + + -- dimensions + country_name, + country_short_code, + country_phone_code, + state_name, + district_name, + postcode, + address_line_one, + address_line_two, + is_billing_address, + google_returned_address, + phone_number, + location_type, + operational_status, + latitude, + longitude, + location_website, + location_overall_rating, + location_number_of_reviews, + monday_operating_hours, + tuesday_operating_hours, + wednesday_operating_hours, + thursday_operating_hours, + friday_operating_hours, + saturday_operating_hours, + sunday_operating_hours, + json_place_details_full_return, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM lastest_billing_and_non_billing_addresses +) + +SELECT * FROM final__int_exchange__latest_company_billing_addresses \ No newline at end of file From 00e67c55c7812365ccf0c75a080881bac288cb0c Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Mon, 17 Jul 2023 05:16:37 +0000 Subject: [PATCH 10/11] Update company_adds increamental bug --- models/staging/exchange/stg_exchange__company_addresses.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/exchange/stg_exchange__company_addresses.sql b/models/staging/exchange/stg_exchange__company_addresses.sql index 128e2ca..a16c037 100644 --- a/models/staging/exchange/stg_exchange__company_addresses.sql +++ b/models/staging/exchange/stg_exchange__company_addresses.sql @@ -7,7 +7,7 @@ {{ config( materialized='incremental', - unique_key='company_id' + unique_key='company_address_id' ) }} From c58e22bdfac2d7a6d4b56145f40f2643f1f3a275 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Mon, 17 Jul 2023 05:17:50 +0000 Subject: [PATCH 11/11] Update company_adds increamental bug --- models/marts/warehouse/dim_exchange__companies.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/warehouse/dim_exchange__companies.sql b/models/marts/warehouse/dim_exchange__companies.sql index ce13260..6d0e304 100644 --- a/models/marts/warehouse/dim_exchange__companies.sql +++ b/models/marts/warehouse/dim_exchange__companies.sql @@ -230,4 +230,4 @@ final__dim_exchange__companies AS ( FROM companies_rename_and_join_table ) -SELECT *FROM final__dim_exchange__companies \ No newline at end of file +SELECT * FROM final__dim_exchange__companies \ No newline at end of file