Merge branch 'compile_company_dimension_for_dedupe' into 'main'

Compile company dimension for dedupe

See merge request cief-data/dbt_cloud!11

README.md

@@ -118,7 +118,7 @@ Short naming convention lists that need to follow
 
 - Use names based on the _business_ terminology, rather than the source terminology.
 
-- Each model should have a primary key that can identify the unique row, and should be named `<object>_id`, e.g. `account_id` – this makes it easier to know what `id` is being referenced in downstream joined models.
+- Each model should have a primary key that can identify the unique row, and should be named `<object>_id`, e.g. `account_id` - this makes it easier to know what `id` is being referenced in downstream joined models.
 
 - If a surrogate key is created, it should be named `<object>_sk`.
 

models/intermediate/int_exchange__latest_company_billing_addresses.sql

@@ -0,0 +1,84 @@
+-- IMPORTS
+WITH company_addresses AS (
+    SELECT * FROM {{ ref('stg_exchange__company_addresses') }}
+),
+
+-- LOGIC
+remove_deleted_addresses AS (
+    SELECT 
+        *
+
+    FROM company_addresses
+
+    WHERE 
+        deleted_datetime IS NULL
+),
+
+lastest_billing_and_non_billing_addresses AS (
+    SELECT 
+        *,
+        -- 1 will be latest updated address, rank by updated_at date
+        row_number() OVER 
+                    (PARTITION BY remove_deleted_addresses.company_id, remove_deleted_addresses.is_billing_address
+                    ORDER BY remove_deleted_addresses.updated_datetime DESC)                 AS lastest_address_rank
+
+    FROM remove_deleted_addresses
+    
+    QUALIFY
+        lastest_address_rank = 1
+),
+
+-- FINAL
+final__int_exchange__latest_company_billing_addresses AS (
+
+    SELECT
+        -- ids
+        company_address_id,
+        company_id,
+        country_id,
+        state_id,
+        district_id,
+        google_place_id,
+
+        -- dimensions
+        country_name,
+        country_short_code,
+        country_phone_code,
+        state_name,     
+        district_name,
+        postcode,
+        address_line_one,
+        address_line_two,
+        is_billing_address,
+        google_returned_address,
+        phone_number,
+        location_type,
+        operational_status,
+        latitude,
+        longitude,
+        location_website,
+        location_overall_rating,
+        location_number_of_reviews,
+        monday_operating_hours,
+        tuesday_operating_hours,
+        wednesday_operating_hours,
+        thursday_operating_hours,
+        friday_operating_hours,
+        saturday_operating_hours,
+        sunday_operating_hours,
+        json_place_details_full_return,
+
+        -- measures
+
+        -- date/times
+        deleted_datetime,
+        created_datetime,
+        updated_datetime,
+
+        -- metadata
+         '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime 
+
+    FROM lastest_billing_and_non_billing_addresses
+) 
+
+SELECT * FROM final__int_exchange__latest_company_billing_addresses

models/marts/reporting/rep_exchange__company_lists.sql

@@ -21,7 +21,7 @@ company_agg AS (
     companies.company_marking_id,
     companies.company_created_datetime,
     COUNT(transaction_orders.transaction_order_id) AS count_order,
-    COUNT(IFF(transaction_orders.order_status = 'completed', transaction_orders.transaction_order_id, null)) AS count_completed_order,
+    COUNT(IFF(transaction_orders.order_status = 'COMPLETED', transaction_orders.transaction_order_id, null)) AS count_completed_order,
     MAX(transaction_orders.order_created_datetime)  AS last_order_datetime,
     COALESCE(last_order_datetime, company_created_datetime) AS last_activity_datetime,
     IFF(companies.company_created_datetime < '2021-07-01', 1, 0) AS is_wrong_created_datetime_company,

models/marts/warehouse/dim_exchange__companies.sql

@@ -4,7 +4,7 @@ WITH companies AS (
 ),
 
 company_addresses AS (
-    SELECT * FROM {{ ref('stg_exchange__company_addresses') }}
+    SELECT * FROM {{ ref('int_exchange__latest_company_billing_addresses') }}
 ),
 
 company_wallets AS (
@@ -230,4 +230,4 @@ final__dim_exchange__companies AS (
     FROM companies_rename_and_join_table 
 )
 
-SELECT * FROM final__dim_exchange__companies
+SELECT * FROM final__dim_exchange__companies

models/staging/exchange/_exchange__sources.yml

@@ -21,7 +21,56 @@ sources:
       - daily 
 
 
-    tables: 
+    tables:
+      - name: account_statements
+        description: ""
+        columns:
+          - name: id
+            description: Primary key for 'account_statements'
+      
+
+      - name: seasonal_segment
+        description: Use to assign company into temporary segment, this function normally use in campaign
+        columns:
+          - name: id
+            description: Primary key for 'seasonal_segment'      
+
+
+      - name: statement_accounts
+        description: ""
+        columns:
+          - name: id
+            description: Primary key for 'statement_accounts'
+
+
+      - name: statement_transactions    
+        description: ""
+        columns:
+          - name: id
+            description: Primary key for 'statement_transactions'       
+
+
+      - name: statement_transaction_owners
+        description: ""
+        columns:
+          - name: id
+            description: Primary key for 'statement_transaction_owners'
+
+
+      - name: vouchers
+        description: ""
+        columns:
+          - name: id
+            description: Primary key for 'vouchers'
+
+
+      - name: voucher_redemptions
+        description: ""
+        columns:
+          - name: id
+            description: Primary key for 'voucher_redemptions'        
+
+
       - name: activity_log
         description: Contains most table's activity log that CUD (created, updated, deleted) in www.exchange.cief-malaysia.com
         columns:

models/staging/exchange/base/base_exchange__company_addresses.sql

@@ -24,30 +24,6 @@ addresses_rename AS (
     FROM addresses
 ),
 
-remove_deleted_addresses AS (
-    SELECT 
-        *
-
-    FROM addresses_rename
-
-    WHERE 
-        deleted_datetime IS NULL
-),
-
-lastest_billing_and_non_billing_addresses AS (
-    SELECT 
-        *,
-        -- 1 will be latest updated address, rank by updated_at date
-        row_number() OVER 
-                    (PARTITION BY remove_deleted_addresses.company_id, remove_deleted_addresses.is_billing_address
-                    ORDER BY remove_deleted_addresses.updated_datetime DESC)                 AS lastest_address_rank
-
-    FROM remove_deleted_addresses
-    
-    QUALIFY
-        lastest_address_rank = 1
-),
-
 -- FINAL
 final__base_exchange__company_addresses AS (
 
@@ -75,7 +51,7 @@ final__base_exchange__company_addresses AS (
         -- metadata
         _dbt_ran_datetime
 
-    FROM lastest_billing_and_non_billing_addresses
+    FROM addresses_rename
 ) 
 
 SELECT * FROM final__base_exchange__company_addresses

models/staging/exchange/stg_exchange__company_addresses.sql

@@ -7,7 +7,7 @@
 {{
     config(
         materialized='incremental',
-        unique_key='company_id'
+        unique_key='company_address_id'
     )
 }}
 

models/staging/googlesheet/_googlesheet__docs.md

@@ -0,0 +1,17 @@
+{% docs src_googlesheet_airbyte %}
+
+Application link: https://airbyte.com/
+
+Architecture: Sales Interface Sheet -> AppScript -> Clone Centralisations Sheet -> Airbyte -> Snowflake
+
+Data is extract from googlesheet [gsheet_clone_centralisations](https://docs.google.com/spreadsheets/d/1GD2l-QnbLY-CfC3BCAouNCM3ROzboCKgeRXEhl64NUg/edit#gid=0), using airbyte.
+
+Before load into 'gsheet_clone_centralisations' it will clone the data from sales gsheet interface [sales_enq_recoarding](https://docs.google.com/spreadsheets/d/1ENYzT_cihpFiNfc7DRZpSRf-PXi_R6cmRn9XKY8a_Vk/edit#gid=376414069)
+
+AppScript to clone the data before airbyte clone into snowflake [AppScript](https://script.google.com/u/1/home/projects/19J_a0P4FB-DVkNGz0THleNp7ISXZxrh8eyNvKxZyoXTP8DOJMWz1VsiK)
+
+- `_airbyte_emitted_at` will add into the raw data by airbyte when extract, it is UTC timezone
+
+- Every 11pm Malaysia Timezone, AppScript will clone the data into Clone Centralisations Sheet and 12am Malaysia Timezone airbyte will clone the data into snowflake
+
+{% enddocs %}

models/staging/googlesheet/_googlesheet__sources.yml

@@ -0,0 +1,50 @@
+version: 2
+
+sources:
+  - name: src_googlesheet_airbyte
+    description: '{{ doc("src_googlesheet_airbyte") }}'
+    database: DEV_CIEF_RAW_DB
+    schema: GOOGLESHEET_AIRBYTE
+    loader: Airbyte
+    loaded_at_field: _airbyte_emitted_at
+    freshness:
+      warn_after: {count: 26, period: hour}
+      error_after: {count: 48, period: hour}
+    meta: 
+      owner: "@yam"
+      model_maturity: prod
+    tags:
+      - googlesheet
+      - daily 
+
+
+    tables: 
+      - name: _airbyte_raw_leads_activations
+        description: This data is get from sales gsheet, sales will call user once they register & not put any order
+        columns:
+          - name: _airbyte_ab_id
+            description: Primary key for '_airbyte_raw_leads_activations'
+            tests:
+              - unique
+              - not_null
+
+          - name: _airbyte_data
+            description: JSON data which come from gsheet
+            tests:
+              - not_null
+
+
+      - name: _airbyte_raw_event_and_holiday_lists
+        description: Data retrieved from gsheet, comprises dates for events and holidays that may or may not impact operation of company.
+        columns:
+          - name: _airbyte_ab_id
+            description: Primary key for '_airbyte_raw_event_and_holiday_lists'
+            tests: 
+              - unique
+              - not_null
+          
+          - name: _airbyte_data
+            description: JSON data which come from gsheet
+            tests:
+              - not_null
+