diff --git a/README.md b/README.md index 3bead35..ca33c80 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Short naming convention lists that need to follow - Use names based on the _business_ terminology, rather than the source terminology. -- Each model should have a primary key that can identify the unique row, and should be named `_id`, e.g. `account_id` – this makes it easier to know what `id` is being referenced in downstream joined models. +- Each model should have a primary key that can identify the unique row, and should be named `_id`, e.g. `account_id` - this makes it easier to know what `id` is being referenced in downstream joined models. - If a surrogate key is created, it should be named `_sk`. diff --git a/models/intermediate/int_exchange__latest_company_billing_addresses.sql b/models/intermediate/int_exchange__latest_company_billing_addresses.sql new file mode 100644 index 0000000..e94daff --- /dev/null +++ b/models/intermediate/int_exchange__latest_company_billing_addresses.sql @@ -0,0 +1,84 @@ +-- IMPORTS +WITH company_addresses AS ( + SELECT * FROM {{ ref('stg_exchange__company_addresses') }} +), + +-- LOGIC +remove_deleted_addresses AS ( + SELECT + * + + FROM company_addresses + + WHERE + deleted_datetime IS NULL +), + +lastest_billing_and_non_billing_addresses AS ( + SELECT + *, + -- 1 will be latest updated address, rank by updated_at date + row_number() OVER + (PARTITION BY remove_deleted_addresses.company_id, remove_deleted_addresses.is_billing_address + ORDER BY remove_deleted_addresses.updated_datetime DESC) AS lastest_address_rank + + FROM remove_deleted_addresses + + QUALIFY + lastest_address_rank = 1 +), + +-- FINAL +final__int_exchange__latest_company_billing_addresses AS ( + + SELECT + -- ids + company_address_id, + company_id, + country_id, + state_id, + district_id, + google_place_id, + + -- dimensions + country_name, + country_short_code, + country_phone_code, + state_name, + district_name, + postcode, + address_line_one, + address_line_two, + is_billing_address, + google_returned_address, + phone_number, + location_type, + operational_status, + latitude, + longitude, + location_website, + location_overall_rating, + location_number_of_reviews, + monday_operating_hours, + tuesday_operating_hours, + wednesday_operating_hours, + thursday_operating_hours, + friday_operating_hours, + saturday_operating_hours, + sunday_operating_hours, + json_place_details_full_return, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM lastest_billing_and_non_billing_addresses +) + +SELECT * FROM final__int_exchange__latest_company_billing_addresses \ No newline at end of file diff --git a/models/marts/reporting/rep_exchange__company_lists.sql b/models/marts/reporting/rep_exchange__company_lists.sql index af2131c..793ac3d 100644 --- a/models/marts/reporting/rep_exchange__company_lists.sql +++ b/models/marts/reporting/rep_exchange__company_lists.sql @@ -21,7 +21,7 @@ company_agg AS ( companies.company_marking_id, companies.company_created_datetime, COUNT(transaction_orders.transaction_order_id) AS count_order, - COUNT(IFF(transaction_orders.order_status = 'completed', transaction_orders.transaction_order_id, null)) AS count_completed_order, + COUNT(IFF(transaction_orders.order_status = 'COMPLETED', transaction_orders.transaction_order_id, null)) AS count_completed_order, MAX(transaction_orders.order_created_datetime) AS last_order_datetime, COALESCE(last_order_datetime, company_created_datetime) AS last_activity_datetime, IFF(companies.company_created_datetime < '2021-07-01', 1, 0) AS is_wrong_created_datetime_company, diff --git a/models/marts/warehouse/dim_exchange__companies.sql b/models/marts/warehouse/dim_exchange__companies.sql index d27d5b9..6d0e304 100644 --- a/models/marts/warehouse/dim_exchange__companies.sql +++ b/models/marts/warehouse/dim_exchange__companies.sql @@ -4,7 +4,7 @@ WITH companies AS ( ), company_addresses AS ( - SELECT * FROM {{ ref('stg_exchange__company_addresses') }} + SELECT * FROM {{ ref('int_exchange__latest_company_billing_addresses') }} ), company_wallets AS ( @@ -230,4 +230,4 @@ final__dim_exchange__companies AS ( FROM companies_rename_and_join_table ) -SELECT * FROM final__dim_exchange__companies +SELECT * FROM final__dim_exchange__companies \ No newline at end of file diff --git a/models/staging/exchange/_exchange__sources.yml b/models/staging/exchange/_exchange__sources.yml index f4e3550..20620be 100644 --- a/models/staging/exchange/_exchange__sources.yml +++ b/models/staging/exchange/_exchange__sources.yml @@ -21,7 +21,56 @@ sources: - daily - tables: + tables: + - name: account_statements + description: "" + columns: + - name: id + description: Primary key for 'account_statements' + + + - name: seasonal_segment + description: Use to assign company into temporary segment, this function normally use in campaign + columns: + - name: id + description: Primary key for 'seasonal_segment' + + + - name: statement_accounts + description: "" + columns: + - name: id + description: Primary key for 'statement_accounts' + + + - name: statement_transactions + description: "" + columns: + - name: id + description: Primary key for 'statement_transactions' + + + - name: statement_transaction_owners + description: "" + columns: + - name: id + description: Primary key for 'statement_transaction_owners' + + + - name: vouchers + description: "" + columns: + - name: id + description: Primary key for 'vouchers' + + + - name: voucher_redemptions + description: "" + columns: + - name: id + description: Primary key for 'voucher_redemptions' + + - name: activity_log description: Contains most table's activity log that CUD (created, updated, deleted) in www.exchange.cief-malaysia.com columns: diff --git a/models/staging/exchange/base/base_exchange__company_addresses.sql b/models/staging/exchange/base/base_exchange__company_addresses.sql index a828656..de2a9ec 100644 --- a/models/staging/exchange/base/base_exchange__company_addresses.sql +++ b/models/staging/exchange/base/base_exchange__company_addresses.sql @@ -24,30 +24,6 @@ addresses_rename AS ( FROM addresses ), -remove_deleted_addresses AS ( - SELECT - * - - FROM addresses_rename - - WHERE - deleted_datetime IS NULL -), - -lastest_billing_and_non_billing_addresses AS ( - SELECT - *, - -- 1 will be latest updated address, rank by updated_at date - row_number() OVER - (PARTITION BY remove_deleted_addresses.company_id, remove_deleted_addresses.is_billing_address - ORDER BY remove_deleted_addresses.updated_datetime DESC) AS lastest_address_rank - - FROM remove_deleted_addresses - - QUALIFY - lastest_address_rank = 1 -), - -- FINAL final__base_exchange__company_addresses AS ( @@ -75,7 +51,7 @@ final__base_exchange__company_addresses AS ( -- metadata _dbt_ran_datetime - FROM lastest_billing_and_non_billing_addresses + FROM addresses_rename ) SELECT * FROM final__base_exchange__company_addresses \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__company_addresses.sql b/models/staging/exchange/stg_exchange__company_addresses.sql index 128e2ca..a16c037 100644 --- a/models/staging/exchange/stg_exchange__company_addresses.sql +++ b/models/staging/exchange/stg_exchange__company_addresses.sql @@ -7,7 +7,7 @@ {{ config( materialized='incremental', - unique_key='company_id' + unique_key='company_address_id' ) }} diff --git a/models/staging/googlesheet/_googlesheet__docs.md b/models/staging/googlesheet/_googlesheet__docs.md new file mode 100644 index 0000000..c5c02ce --- /dev/null +++ b/models/staging/googlesheet/_googlesheet__docs.md @@ -0,0 +1,17 @@ +{% docs src_googlesheet_airbyte %} + +Application link: https://airbyte.com/ + +Architecture: Sales Interface Sheet -> AppScript -> Clone Centralisations Sheet -> Airbyte -> Snowflake + +Data is extract from googlesheet [gsheet_clone_centralisations](https://docs.google.com/spreadsheets/d/1GD2l-QnbLY-CfC3BCAouNCM3ROzboCKgeRXEhl64NUg/edit#gid=0), using airbyte. + +Before load into 'gsheet_clone_centralisations' it will clone the data from sales gsheet interface [sales_enq_recoarding](https://docs.google.com/spreadsheets/d/1ENYzT_cihpFiNfc7DRZpSRf-PXi_R6cmRn9XKY8a_Vk/edit#gid=376414069) + +AppScript to clone the data before airbyte clone into snowflake [AppScript](https://script.google.com/u/1/home/projects/19J_a0P4FB-DVkNGz0THleNp7ISXZxrh8eyNvKxZyoXTP8DOJMWz1VsiK) + +- `_airbyte_emitted_at` will add into the raw data by airbyte when extract, it is UTC timezone + +- Every 11pm Malaysia Timezone, AppScript will clone the data into Clone Centralisations Sheet and 12am Malaysia Timezone airbyte will clone the data into snowflake + +{% enddocs %} \ No newline at end of file diff --git a/models/staging/googlesheet/_googlesheet__sources.yml b/models/staging/googlesheet/_googlesheet__sources.yml new file mode 100644 index 0000000..800c970 --- /dev/null +++ b/models/staging/googlesheet/_googlesheet__sources.yml @@ -0,0 +1,50 @@ +version: 2 + +sources: + - name: src_googlesheet_airbyte + description: '{{ doc("src_googlesheet_airbyte") }}' + database: DEV_CIEF_RAW_DB + schema: GOOGLESHEET_AIRBYTE + loader: Airbyte + loaded_at_field: _airbyte_emitted_at + freshness: + warn_after: {count: 26, period: hour} + error_after: {count: 48, period: hour} + meta: + owner: "@yam" + model_maturity: prod + tags: + - googlesheet + - daily + + + tables: + - name: _airbyte_raw_leads_activations + description: This data is get from sales gsheet, sales will call user once they register & not put any order + columns: + - name: _airbyte_ab_id + description: Primary key for '_airbyte_raw_leads_activations' + tests: + - unique + - not_null + + - name: _airbyte_data + description: JSON data which come from gsheet + tests: + - not_null + + + - name: _airbyte_raw_event_and_holiday_lists + description: Data retrieved from gsheet, comprises dates for events and holidays that may or may not impact operation of company. + columns: + - name: _airbyte_ab_id + description: Primary key for '_airbyte_raw_event_and_holiday_lists' + tests: + - unique + - not_null + + - name: _airbyte_data + description: JSON data which come from gsheet + tests: + - not_null + diff --git a/models/staging/googlesheet/base/.gitkeep b/models/staging/googlesheet/base/.gitkeep new file mode 100644 index 0000000..e69de29