Crisp Extract Load

crisp/extract/crisp_extract_function.py

@@ -14,23 +14,25 @@ def connect_crisp(config):
         print("[FUNCTION_ERROR]-connect_crisp")
         return False
     
-def list_website_operators_csv(website_id, client, path, filename):
+def list_website_operators_json(website_id, client, path, filename):
     try:
         output_location = path + filename
         list_website_operators_json = client.website.list_website_operators(website_id)
         list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json, sep='__')
 
         # Datetime add
-        list_website_operators_df['api_called_at'] = date.today() - timedelta(days = 1)
+        list_website_operators_df['api_called_at'] = datetime.now()
 
-        list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        # list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        list_website_operators_df.to_json(output_location, orient='records', force_ascii = False)
         print("[Download Complete]",filename)
+
     except Exception as e:
         print(e)
-        print("[FUNCTION_ERROR]-list_website_operators_csv")
+        print("[FUNCTION_ERROR]-list_website_operators_json")
         return False         
 
-def list_people_profiles_csv(website_id, client, path, filename):
+def list_people_profiles_json(website_id, client, path, filename):
     try:
         output_location = path + filename
         list_people_profiles_df = pd.DataFrame()
@@ -53,23 +55,24 @@ def list_people_profiles_csv(website_id, client, path, filename):
         list_people_profiles_df['active__last']=(pd.to_datetime(list_people_profiles_df['active__last'],unit='ms')) 
 
         # Datetime add
-        list_people_profiles_df['api_called_at'] = date.today() - timedelta(days = 1)
+        list_people_profiles_df['api_called_at'] = datetime.now()
 
-        list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        # list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        list_people_profiles_df.to_json(output_location, orient='records', force_ascii = False)
         print("[Download Complete]",filename)
 
     except Exception as e:
         print(e)
-        print("[FUNCTION_ERROR]-list_people_profiles_csv")
+        print("[FUNCTION_ERROR]-list_people_profiles_json")
         return False         
 
 
-def get_people_data_csv(website_id, client, path, filename, list_people_profiles_csv_path):
+def get_people_data_json(website_id, client, path, filename, list_people_profiles_json_path):
     try:
         output_location = path + filename
         get_people_data_df = pd.DataFrame()
 
-        person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
+        person_ids = pd.read_json(list_people_profiles_json_path)['people_id']
 
         # Get marking by CustomerService Operator
         for people_id in person_ids:
@@ -85,40 +88,104 @@ def get_people_data_csv(website_id, client, path, filename, list_people_profiles
         get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "__")
 
         # Datetime add
-        get_people_data_df['api_called_at'] = date.today() - timedelta(days = 1)
+        get_people_data_df['api_called_at'] = datetime.now()
 
-        get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        # get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        get_people_data_df.to_json(output_location, orient='records', force_ascii = False)
         print("[Download Complete]",filename)
 
     except Exception as e:
         print(e)
-        print("[FUNCTION_ERROR]-get_people_data_csv")
+        print("[FUNCTION_ERROR]-get_people_data_json")
         return False         
 
 
 
-def get_people_subscription_status_csv(website_id, client, path, filename, list_people_profiles_csv_path):
+def get_people_subscription_status_json(website_id, client, path, filename, list_people_profiles_json_path):
     try:
         output_location = path + filename
         get_people_subscription_status_df = pd.DataFrame()
 
-        person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
+        person_ids = pd.read_json(list_people_profiles_json_path)['people_id']
         
         # Get email subscription status
         for people_id in person_ids:
-            pget_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
+            get_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
-            temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json, sep='__')
+            temp_df = general_function.json_normalize_to_df(get_people_subscription_status_df_json, sep='__')
             temp_df['people_id'] = people_id
             get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True)
 
         # Datetime add
-        get_people_subscription_status_df['api_called_at'] = date.today() - timedelta(days = 1)
+        get_people_subscription_status_df['api_called_at'] = datetime.now()
 
         get_people_subscription_status_df.columns = get_people_subscription_status_df.columns.str.replace(".", "-")
-        get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        # get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        get_people_subscription_status_df.to_json(output_location, orient='records', force_ascii = False)
+
         print("[Download Complete]",filename)
 
     except Exception as e:
         print(e)
-        print("[FUNCTION_ERROR]-get_people_subscription_status_csv")
+        print("[FUNCTION_ERROR]-get_people_subscription_status_json")
+        return False         
+
+
+
+def list_conversations_json(website_id, client, path, filename):
+    try:
+        loop_flag = True
+        page_number = 1
+        output_location = path + filename
+        conversations_list_df = pd.DataFrame()
+
+        while loop_flag:
+            conversations_list_df_json = client.website.list_conversations(website_id, page_number)
+
+            temp_df = general_function.json_normalize_to_df(conversations_list_df_json, sep='__')
+            conversations_list_df = pd.concat([conversations_list_df, temp_df], ignore_index=True)
+
+            if conversations_list_df_json == []:
+                loop_flag = False
+            else:
+                page_number += 1
+        
+        conversations_list_df.columns = conversations_list_df.columns.str.replace("-", "_")
+        conversations_list_df['api_called_at'] = datetime.now()
+
+        conversations_list_df.to_json(output_location, orient='records', force_ascii = False)
+        print("[Download Complete]",filename)
+        
+    except Exception as e:
+        print(e)
+        print("[FUNCTION_ERROR]-list_conversations_json")
+        return False  
+
+
+def get_message_in_conversations_json(website_id, client, path, filename, list_conversations_json_path):
+    try:
+        output_location = path + filename
+        get_message_in_conversations_df = pd.DataFrame()
+        session_ids = pd.read_json(list_conversations_json_path)['session_id']
+
+        for session_id in session_ids:
+            print(session_id) #NEED DELETE
+
+            get_message_in_conversations_json = client.website.get_messages_in_conversation(website_id, session_id, "")   
+            temp_df = general_function.json_normalize_to_df(get_message_in_conversations_json, sep='__')
+            get_message_in_conversations_df = pd.concat([temp_df, get_message_in_conversations_df], ignore_index=True)
+
+            while (temp_df.shape[0] == 40):       #Maximum can get 40 conversation per request, Request for date before if the temp_df is full
+                query = { "timestamp_before" : str(temp_df.at[0,'timestamp']) }
+                get_message_in_conversations_json = client.website.get_messages_in_conversation(website_id, session_id, query)
+                temp_df = pd.json_normalize(get_message_in_conversations_json, sep='__')
+                get_message_in_conversations_df = pd.concat([temp_df,get_message_in_conversations_df], ignore_index=True)
+            
+        get_message_in_conversations_df['api_called_at'] = datetime.now()
+        get_message_in_conversations_df.to_json(output_location, orient='records', force_ascii = False)
+        print("[Download Complete]",filename)
+        
+
+    except Exception as e:
+        print(e)
+        print("[FUNCTION_ERROR]-get_message_in_conversations_json")
         return False 

main.py

@@ -1,5 +1,6 @@
 import os
 from os import walk
+import pandas as pd
 import general_function
 import exchange.extract.exchange_extract_function as exchange_extract_func
 import exchange.load.exchange_load_function as exchange_load_func
@@ -92,16 +93,22 @@ def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
     CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG)
 
     #Get operators csv
-    crisp_extract_func.list_website_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_website_operators.csv')
+    crisp_extract_func.list_website_operators_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_website_operators.json')
 
     #Get users(people) csv
-    crisp_extract_func.list_people_profiles_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_people_profiles.csv')  
+    crisp_extract_func.list_people_profiles_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_people_profiles.json')  
 
-    # Get peoples manually marking by operator
+    #Get peoples manually marking by operator
-    crisp_extract_func.get_people_data_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv')
+    crisp_extract_func.get_people_data_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.json')
 
-    # Get peoples subscription status
+    #Get peoples subscription status
-    crisp_extract_func.get_people_subscription_status_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv')
+    crisp_extract_func.get_people_subscription_status_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.json')
+
+    #Get current converstation list
+    crisp_extract_func.list_conversations_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_conversations.json')
+
+    #Get message in conversations
+    crisp_extract_func.get_message_in_conversations_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_message_in_conversations.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_conversations.json')
 
 def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
     #Config file name