data-management/crisp/extract/crisp_extract_function.py

from crisp_api import Crisp
import general_function
import pandas as pd

def connect_crisp(config):
    try:
        client = Crisp()
        client.set_tier("plugin")
        client.authenticate(config["identifier"], config["key"])
        return client
    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-connect_crisp")
        return False
    
def get_operators_csv(website_id, client, path, filename):
    try:
        output_location = path + filename
        operators_list_json = client.website.list_website_operators(website_id)
        operators_list_df = general_function.json_normalize_to_df(operators_list_json)
        operators_list_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        print("[Download Complete]",filename)
    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-get_operators_csv")
        return False         

def get_peoples_csv(website_id, client, path, filename):
    try:
        output_location = path + filename
        peoples_list_df = pd.DataFrame()
        peoples_data_df = pd.DataFrame()
        loop_flag = True
        page_number = 1

        while loop_flag == True:
            peoples_list_json = client.website.list_people_profiles(website_id, page_number)
            temp_df = general_function.json_normalize_to_df(peoples_list_json)
            peoples_list_df = pd.concat([temp_df,peoples_list_df], ignore_index=True)
            if not peoples_list_json:
                loop_flag = False
            else:
                page_number += 1

        for people_id in peoples_list_df.people_id:
            print(people_id)
            people_data_json = client.website.get_people_data(website_id, people_id)
            temp_df = general_function.json_normalize_to_df(people_data_json)
            temp_df['people_id'] = people_id
            peoples_data_df = pd.concat([temp_df,peoples_data_df], ignore_index=True)

        peoples_list_df = peoples_list_df.merge(peoples_data_df, on='people_id', how='left')

        peoples_list_df.to_csv(output_location, sep=',', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        peoples_data_df.to_csv('test.csv', sep=',', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        print("[Download Complete]",filename)

    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-get_peoples_csv")
        return False