Merge branch 'compile_company_dimension_for_dedupe' into 'main'

Compile company dimension for dedupe

See merge request cief-data/dbt_cloud!19

dbt_project.yml

@@ -35,16 +35,6 @@ models:
     staging:
       +tags:
         - staging
-
-      autocount:
-          +materialized: view
-          +schema: autocount__staging
-          +tags:
-            - autocount
-
-          base:
-            +materialized: view
-            +schema: autocount__base    
     
       crisp:
           +materialized: view
@@ -64,7 +54,15 @@ models:
 
           base:
             +materialized: view
-            +schema: exchange__base              
+            +schema: exchange__base      
+
+      googlesheet:
+          +materialized: view
+          +schema: googlesheet__stagting
+
+          base:
+            +materialized: view
+            +schema: googlesheet__base
 
       shipping:
           +materialized: view

models/intermediate/int__dates.sql

@@ -0,0 +1,250 @@
+-- CONFIG
+{{
+    config(
+        materialized='view'
+    )
+}}
+
+-- IMPORT
+WITH holiday_and_non_working_day_lists AS (
+    SELECT * FROM {{ ref('stg_googlesheet__event_and_holiday_lists') }}
+),
+
+-- LOGIC
+date_spine AS (
+
+  {{ dbt_utils.date_spine(
+      start_date="to_date('01/01/2015', 'mm/dd/yyyy')",
+      datepart="day",
+      end_date="dateadd(year, 40, current_date)"
+     )
+  }}
+
+),
+
+fiscal_calendar as (
+
+    SELECT
+        date_day,
+        date_day                                                                                AS date_actual,
+
+        DAYNAME(date_day)                                                                       AS day_name,
+
+        DATE_PART('month', date_day)                                                            AS month_actual,
+        DATE_PART('year', date_day)                                                             AS year_actual,
+
+        DATE_PART(quarter, date_day)                                                            AS quarter_actual,
+
+        CASE WHEN DATE_PART(dayofweek, date_day) = 0 THEN 7
+        ELSE  DATE_PART(dayofweek, date_day) END                                                AS day_of_week,
+
+        CASE WHEN day_name = 'Mon' THEN date_day
+        ELSE DATEADD('day', 0, DATE_TRUNC('week', date_day)) END                                AS first_day_of_week,
+
+        CASE WHEN day_name = 'Mon' THEN WEEK(date_day)
+        ELSE WEEK(date_day) END                                                                 AS week_of_year_temp, --remove this column
+
+        CASE WHEN day_name = 'Mon' AND LEAD(week_of_year_temp) OVER (ORDER BY date_day) = '1'
+        THEN '1'
+        ELSE week_of_year_temp END                                                              AS week_of_year,
+
+        DATE_PART('day', date_day)                                                              AS day_of_month,
+
+        ROW_NUMBER() OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day)          AS day_of_quarter,
+        ROW_NUMBER() OVER (PARTITION BY year_actual ORDER BY date_day)                          AS day_of_year,
+
+        -- @TODO @GERSON ,done
+        -- BUSINESS LOGIC: Feb - Jan (2022), Feb - Jan (2023)
+        CASE 
+            WHEN month_actual  in   (2,3,4,5,6,7,8,9,10,11,12) THEN year_actual
+            WHEN DATE_PART('month', DATEADD(  month,-1, date_day) )  = 12 THEN year_actual-1
+            ELSE (year_actual+1) 
+        END                                                                                     AS fiscal_year,
+
+        -- @TODO @GERSON , done
+        -- BUSINESS LOGIC: Feb - Apr (Q1), May - Jul (Q2), Aug - Oct (Q3), Nov - Jan (Q4)
+        CASE WHEN  month_actual in  (2,3,4)   THEN '1'
+        WHEN month_actual  in  (5,6,7)      THEN '2'
+        WHEN month_actual in  (8,9,10)      THEN '3'
+        WHEN month_actual in (11,12)        THEN '4'
+        WHEN DATE_PART('month', DATEADD(month,-1,date_day)) =12 THEN '4'
+        -- WHEN  month_actual=  DATE_PART('year', month_actual) +1   and month_actual=1 THEN '4'
+        ELSE '4' END                                                                            AS fiscal_quarter,
+
+        -- @TODO @GERSON     ,done
+        ROW_NUMBER() OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day)          AS day_of_fiscal_quarter,
+        ROW_NUMBER() OVER (PARTITION BY fiscal_year ORDER BY date_day)                          AS day_of_fiscal_year,
+
+        TO_CHAR(date_day, 'MMMM')                                                               AS month_name,
+
+        TRUNC(date_day, 'Month')                                                                AS first_day_of_month,
+        LAST_VALUE(date_day) OVER (PARTITION BY year_actual, month_actual ORDER BY date_day)    AS last_day_of_month,
+
+        FIRST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day)                 AS first_day_of_year,
+        LAST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day)                  AS last_day_of_year,
+
+        FIRST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS first_day_of_quarter,
+        LAST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day)  AS last_day_of_quarter,
+
+        -- @TODO @GERSON ,done
+        -- Change this 2 too 
+        FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS first_day_of_fiscal_quarter,
+        LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day)  AS last_day_of_fiscal_quarter,
+
+        -- @TODO @GERSON,done
+        FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day)                 AS first_day_of_fiscal_year,
+        LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day)                  AS last_day_of_fiscal_year,
+
+        -- @TODO @GERSON ,done
+        DATEDIFF('week', first_day_of_fiscal_year, date_actual)+1                               AS week_of_fiscal_year,
+
+        -- @TODO @GERSON ,done
+        CASE WHEN EXTRACT('month', date_day) = 1 THEN 12
+        ELSE EXTRACT('month', date_day) - 1 END                                                 AS month_of_fiscal_year,
+
+        LAST_VALUE(date_day) OVER (PARTITION BY first_day_of_week ORDER BY date_day)            AS last_day_of_week,
+        
+        (year_actual || '-Q' || fiscal_quarter)                                                 AS quarter_name,
+
+        -- @TODO @GERSON,done
+        (fiscal_year || '-' || DECODE(fiscal_quarter,
+        1, 'Q1',
+        2, 'Q2',
+        3, 'Q3',
+        4, 'Q4'))                                                                               AS fiscal_quarter_name,
+        ('FY' || SUBSTR(fiscal_quarter_name, 3, 7))                                             AS fiscal_quarter_name_fy,
+        DENSE_RANK() OVER (ORDER BY fiscal_quarter_name)                                        AS fiscal_quarter_number_absolute,
+        --here check
+        fiscal_year || '-' || MONTHNAME(date_day)                                               AS fiscal_month_name,
+        ('FY' || SUBSTR(fiscal_month_name, 3, 8))                                               AS fiscal_month_name_fy,
+
+        -- @TODO @GERSON
+        DATE_TRUNC('month', last_day_of_fiscal_quarter)                                         AS last_month_of_fiscal_quarter,
+        IFF(DATE_TRUNC('month', last_day_of_fiscal_quarter) = date_actual, TRUE, FALSE)         AS is_first_day_of_last_month_of_fiscal_quarter,
+        DATE_TRUNC('month', last_day_of_fiscal_year)                                            AS last_month_of_fiscal_year,
+        IFF(DATE_TRUNC('month', last_day_of_fiscal_year) = date_actual, TRUE, FALSE)            AS is_first_day_of_last_month_of_fiscal_year,
+
+        DATEADD('day',7,DATEADD('month',1,first_day_of_month))                                  AS snapshot_date_fpa,
+        DATEADD('day',44,DATEADD('month',1,first_day_of_month))                                 AS snapshot_date_billings
+
+    FROM date_spine
+
+    QUALIFY date_actual >= '2016-01-01'
+
+    ORDER BY date_day
+),
+
+holiday_group_by_date AS (
+
+    SELECT 
+        date,
+        day,    
+        ARRAY_AGG(DISTINCT event_name) WITHIN GROUP (ORDER BY event_name)                               AS holiday_event_names,
+        ARRAY_AGG(DISTINCT event_description) WITHIN GROUP (ORDER BY event_description)                 AS holiday_event_descriptions,
+        ARRAY_AGG(DISTINCT state) WITHIN GROUP (ORDER BY state)                                         AS holiday_states,
+        ARRAY_AGG(DISTINCT country) WITHIN GROUP (ORDER BY country)                                     AS holiday_countries,
+        ARRAY_AGG(DISTINCT religion) WITHIN GROUP (ORDER BY religion)                                   AS holiday_religions,
+        ARRAY_AGG(DISTINCT holiday_general_type) WITHIN GROUP (ORDER BY holiday_general_type)           AS holiday_event_general_types,
+        ARRAY_AGG(DISTINCT holiday_specific_type) WITHIN GROUP (ORDER BY holiday_specific_type)         AS holiday_event_specific_types,
+        MAX(is_malaysia_holiday)                                                                        AS is_malaysia_holiday,
+        MAX(is_company_holiday)                                                                         AS is_company_holiday,
+        MAX(is_china_holiday)                                                                           AS is_china_holiday,
+        MAX(is_china_warehouse_holiday)                                                                 AS is_china_warehouse_holiday
+
+    FROM holiday_and_non_working_day_lists 
+
+    GROUP BY 
+        day,
+        date
+
+    ORDER BY date
+),
+
+fiscal_calendar_join_holiday_list AS (
+
+    SELECT
+        fiscal_calendar.*,
+
+        holidays.holiday_event_names,
+        holidays.holiday_event_descriptions,
+        holidays.holiday_event_general_types,
+        holidays.holiday_event_specific_types,
+        holidays.holiday_states,
+        holidays.holiday_countries,
+        holidays.holiday_religions,
+        holidays.is_malaysia_holiday,
+        holidays.is_company_holiday,
+        holidays.is_china_holiday,
+        holidays.is_china_warehouse_holiday,
+
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}'       AS _dbt_ran_datetime
+
+    FROM fiscal_calendar
+
+    LEFT JOIN holiday_group_by_date AS holidays
+        ON fiscal_calendar.date_day = holidays.date
+),
+
+-- FINAL
+final__int__dates AS (
+
+    SELECT
+        date_day,
+        date_actual,
+        day_name,
+        month_actual,
+        year_actual,
+        quarter_actual,
+        day_of_week,
+        first_day_of_week,
+        week_of_year,
+        day_of_month,
+        day_of_quarter,
+        day_of_year,
+        fiscal_year,
+        fiscal_quarter,
+        day_of_fiscal_quarter,
+        day_of_fiscal_year,
+        month_name,
+        first_day_of_month,
+        last_day_of_month,
+        first_day_of_year,
+        last_day_of_year,
+        first_day_of_quarter,
+        last_day_of_quarter,
+        first_day_of_fiscal_quarter,
+        last_day_of_fiscal_quarter,
+        first_day_of_fiscal_year,
+        last_day_of_fiscal_year,
+        week_of_fiscal_year,
+        month_of_fiscal_year,
+        last_day_of_week,
+        quarter_name,
+        fiscal_quarter_name,
+        fiscal_quarter_name_fy,
+        fiscal_quarter_number_absolute,
+        fiscal_month_name,
+        fiscal_month_name_fy,
+        last_month_of_fiscal_quarter,
+        is_first_day_of_last_month_of_fiscal_quarter,
+        last_month_of_fiscal_year,
+        is_first_day_of_last_month_of_fiscal_year,
+        snapshot_date_fpa,
+        snapshot_date_billings,
+        holiday_event_names,
+        holiday_event_descriptions,
+        holiday_event_general_types,
+        holiday_event_specific_types,
+        holiday_countries,
+        holiday_states,
+        holiday_religions,
+        is_malaysia_holiday,
+        is_company_holiday,
+        is_china_holiday,
+        is_china_warehouse_holiday,
+        _dbt_ran_datetime
+
+    FROM fiscal_calendar_join_holiday_list
+)
+
+SELECT * FROM final__int__dates

models/staging/exchange/base/base_exchange__seasonal_segments.sql

@@ -0,0 +1,55 @@
+-- IMPORTS
+WITH source as (
+    SELECT * FROM {{ source('src_exchange_mysql', 'seasonal_segment') }}
+
+),
+
+-- LOGIC
+seasonal_segment_rename AS (
+
+    SELECT
+        id                  AS seasonal_segment_id,
+        company_id,
+        status,
+        segment_id,
+        starting_on         AS seasonal_segment_started_datetime,
+        ending_on           AS seasonal_segment_ended_datetime,
+        deleted_at          AS seasonal_segment_deleted_datetime,
+        created_at          AS seasonal_segment_created_datetime,
+        updated_at          AS seasonal_segment_updated_datetime,
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
+
+    FROM source
+
+    -- id 1 is wrong setup, it should not occur
+    WHERE id != 1
+),
+
+-- FINAL
+final__base_exchange__seasonal_segments AS (
+    
+    SELECT
+        -- ids
+        seasonal_segment_id,
+        company_id,
+        segment_id,
+
+        -- dimensions
+        status,
+
+        -- measures
+
+        -- date/times
+        seasonal_segment_started_datetime,
+        seasonal_segment_ended_datetime,
+        seasonal_segment_deleted_datetime,
+        seasonal_segment_created_datetime,
+        seasonal_segment_updated_datetime,
+
+        -- metadata
+        _dbt_ran_datetime
+
+    FROM seasonal_segment_rename
+)
+
+SELECT * from final__base_exchange__seasonal_segments

models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql

@@ -0,0 +1,78 @@
+-- IMPORT
+WITH holiday_lists AS (
+    SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_event_and_holiday_lists') }}
+),
+
+-- LOGIC
+parse_jason_holidays AS (
+
+    SELECT
+        _airbyte_ab_id              AS airbyte_id,
+        parse_json(_airbyte_data)   AS holiday_json,
+        _airbyte_emitted_at         AS _airbyte_emitted_datetime
+
+    FROM holiday_lists
+),
+
+flatten_json AS (
+
+    SELECT 
+        airbyte_id,
+        holiday_json['id']::INTEGER                             AS holiday_id,
+        holiday_json['date']::DATE                              AS date,
+        holiday_json['day']::STRING                             AS day,
+        holiday_json['event_name']::STRING                      AS event_name,
+        holiday_json['event_desc']::STRING                      AS event_description,
+        holiday_json['general_type']::STRING                    AS holiday_general_type,
+        holiday_json['holiday_type']::STRING                    AS holiday_specific_type,
+        holiday_json['country']::STRING                         AS country,     
+        holiday_json['state']::STRING                           AS state,
+        holiday_json['event_cultural_origin']::STRING           AS religion,
+        holiday_json['is_malaysia_holiday']::INTEGER            AS is_malaysia_holiday,
+        holiday_json['is_company_holiday']::INTEGER             AS is_company_holiday,
+        holiday_json['is_china_holiday']::INTEGER               AS is_china_holiday,
+        holiday_json['is_china_warehouse_holiday']::INTEGER     AS is_china_warehouse_holiday,
+        holiday_json['special_remark']::STRING                  AS remark,
+
+        _airbyte_emitted_datetime,
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
+
+    FROM parse_jason_holidays
+),
+
+-- FINAL
+final__stg_googlesheet__event_and_holiday_lists AS (
+
+    SELECT
+        -- ids
+        holiday_id,
+        airbyte_id,
+
+        -- dimensions
+        day,
+        event_name,
+        event_description,
+        holiday_general_type,
+        holiday_specific_type,
+        country,
+        state,
+        religion,
+        is_malaysia_holiday,
+        is_company_holiday,
+        is_china_holiday,
+        is_china_warehouse_holiday,
+        
+        -- measures
+
+        -- date/times
+        date,
+
+        -- metadata
+        remark,
+        _airbyte_emitted_datetime,
+        _dbt_ran_datetime
+    
+    FROM flatten_json
+)
+
+SELECT * FROM final__stg_googlesheet__event_and_holiday_lists

packages.yml

@@ -20,4 +20,7 @@ packages:
     version: 1.0.0
 
   - package: dbt-labs/codegen
-    version: 0.9.0
+    version: 0.9.0
+
+  - package: calogica/dbt_date
+    version: [">=0.7.0", "<0.8.0"]