Migrate to new airbyte format

models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql

@@ -1,44 +1,36 @@
 -- IMPORT
 WITH holiday_lists AS (
-    SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_event_and_holiday_lists') }}
+    SELECT * FROM {{ source('src_googlesheet_airbyte', 'event_and_holiday_lists') }}
 ),
 
--- LOGIC
-parse_jason_holidays AS (
 
-    SELECT
-        _airbyte_ab_id              AS airbyte_id,
-        parse_json(_airbyte_data)   AS holiday_json,
-        _airbyte_emitted_at         AS _airbyte_emitted_datetime
+-- LOGIC
+data_casting AS (
+
+    SELECT 
+        id                              AS holiday_id,
+        _airbyte_raw_id                 AS _airbyte_id,
+        DATE(date)                      AS date,
+        event_cultural_origin           AS religion,
+        event_desc                      AS event_description,
+        general_type                    AS holiday_general_type,
+        holiday_type                    AS holiday_specific_type,
+        special_remark                  AS remark,
+        day,
+        event_name,
+        country,
+        state,
+        is_malaysia_holiday,
+        is_company_holiday,
+        is_china_holiday,
+        is_china_warehouse_holiday,
+        
+        _airbyte_extracted_at           AS _airbyte_extracted_datetime,
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
 
     FROM holiday_lists
 ),
 
-flatten_json AS (
-
-    SELECT 
-        airbyte_id,
-        holiday_json['id']::INTEGER                             AS holiday_id,
-        holiday_json['date']::DATE                              AS date,
-        holiday_json['day']::STRING                             AS day,
-        holiday_json['event_name']::STRING                      AS event_name,
-        holiday_json['event_desc']::STRING                      AS event_description,
-        holiday_json['general_type']::STRING                    AS holiday_general_type,
-        holiday_json['holiday_type']::STRING                    AS holiday_specific_type,
-        holiday_json['country']::STRING                         AS country,     
-        holiday_json['state']::STRING                           AS state,
-        holiday_json['event_cultural_origin']::STRING           AS religion,
-        holiday_json['is_malaysia_holiday']::INTEGER            AS is_malaysia_holiday,
-        holiday_json['is_company_holiday']::INTEGER             AS is_company_holiday,
-        holiday_json['is_china_holiday']::INTEGER               AS is_china_holiday,
-        holiday_json['is_china_warehouse_holiday']::INTEGER     AS is_china_warehouse_holiday,
-        holiday_json['special_remark']::STRING                  AS remark,
-
-        _airbyte_emitted_datetime,
-        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
-
-    FROM parse_jason_holidays
-),
 
 -- FINAL
 final__stg_googlesheet__event_and_holiday_lists AS (
@@ -46,7 +38,6 @@ final__stg_googlesheet__event_and_holiday_lists AS (
     SELECT
         -- ids
         holiday_id,
-        airbyte_id,
 
         -- dimensions
         day,
@@ -61,6 +52,7 @@ final__stg_googlesheet__event_and_holiday_lists AS (
         is_company_holiday,
         is_china_holiday,
         is_china_warehouse_holiday,
+        remark,
         
         -- measures
 
@@ -68,11 +60,11 @@ final__stg_googlesheet__event_and_holiday_lists AS (
         date,
 
         -- metadata
-        remark,
-        _airbyte_emitted_datetime,
+        _airbyte_id,
+        _airbyte_extracted_datetime,
         _dbt_ran_datetime
     
-    FROM flatten_json
+    FROM data_casting
 )
 
-SELECT * FROM final__stg_googlesheet__event_and_holiday_lists
+SELECT * FROM final__stg_googlesheet__event_and_holiday_lists