From 164e72d1d3e8a8d4acd3ca161c754c17499b5a23 Mon Sep 17 00:00:00 2001
From: Yam ZhengLim <yamzhenglim0427@gmail.com>
Date: Thu, 23 Feb 2023 22:31:41 +0800
Subject: [PATCH] Crisp Extract Load

---
 __pycache__/general_function.cpython-39.pyc   | Bin 1449 -> 1449 bytes
 .../crisp_extract_function.cpython-39.pyc     | Bin 3738 -> 4909 bytes
 crisp/extract/crisp_extract_function.py       | 109 ++++++++++++++----
 .../crisp_load_function.cpython-39.pyc        | Bin 973 -> 973 bytes
 .../exchange_extract_function.cpython-39.pyc  | Bin 1785 -> 1785 bytes
 .../exchange_load_function.cpython-39.pyc     | Bin 985 -> 985 bytes
 main.py                                       |  21 ++--
 .../shipping_extract_function.cpython-39.pyc  | Bin 1785 -> 1785 bytes
 .../shipping_load_function.cpython-39.pyc     | Bin 985 -> 985 bytes
 9 files changed, 102 insertions(+), 28 deletions(-)

diff --git a/__pycache__/general_function.cpython-39.pyc b/__pycache__/general_function.cpython-39.pyc
index a0d8b9156e6bbd3034dd71de92512128b1cd2c6a..d681405a147985f1ac11516895eda9e7f251e094 100644
GIT binary patch
delta 27
hcmZ3<y^@<dk(ZZ?0SKPY{;-jIIwPa==EaO`EC6Mb2V?*M

delta 27
hcmZ3<y^@<dk(ZZ?0SK1HuiVHzosltl^I}Fe764pV2EPCR

diff --git a/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc b/crisp/extract/__pycache__/crisp_extract_function.cpython-39.pyc
index 3754cc16de0dcbd55c52dff8e8e06d9a7fe6b071..cb6096c8f75c266d78508a307b378f92aea4347e 100644
GIT binary patch
literal 4909
zcmbW5O^?*b8OQx@x7}~_&axSn%)US(8ZCrucG(RfQKT70S}7SMV1?ox<TYKM!HwJQ
zDZ7DbxjBR=%7rNB9HJg6hwyFo5beQ=a!D?Hf=Hb5f2vK7r>B8Uy5%anTvaYt{ht5x
zl>J7-mhe0G>3@3mhm!O!x~TpXT>KJ88X;j4lOt&$yRuB@N~E}o$W>QEu15O6aE*cK
zngiX{QAcC??<Ch^1~ZY@n8j+yZDunExx?zLfxOO|tcAS6+Ux}KCR1)o-Oh77QR>Qt
zzRtsRxYbn)jrmzn)Ut38upr9(!lDAY?8#KQ{He(4t=4g*J4nXoGwF#mk#l(}Gx-3c
z$C;Aj4OS(j>Z)|0P}yzirld)ky8pE-O;9pk%d}j1g8G@pG|F#FzrmYNl*bCwZ<c45
zq)Rx%h>b1R7)GPrFfQzn1#uScgaI$q`@#Nr?W@f%x7Kfcaed3XcH_qN8+R`Bk~j`}
znb)JY+wAI`rrNdAAoDUbbL;*nyGNb$#7kd$)C-1Ln8bxX<YAm)It+BN6Ye69LR2i|
z;F-i5=;g)z^()(7rU6g4_x-{6-uCr4*x=!VV0$A-?`O$yd;N=Ro8E?>`M>sKe>WJQ
z`8MX#zc4tw76aQ39%bC`W!vJ)E3dqrQQV`!TpaEfjU#Vj(4=>eNU|auvMsM5ZORt@
zyoJ)9M)%5}g?zqJf564~SGkg_??@9Zm;2IG&ZPr+qGw89&2=U-<*70;GHt3KNV&n(
zt4RAsj<sPrGoGqgCzF}E`UGotTZ%7ZEv^0EVl7RvmS%2>wKSPkSxfV=S*|6{=4)Bo
zDiqIqCKqZN42y<8485KoMFI2ttT1`dOE^n6pDBfra54GLPQrVE=cm0eeDT&-8_B~s
zN__VIdNLSBK^EK@e{>9iC`>c&VQ@DMv%pJ+0r#_nr(Qoz;%==l%RBDL<@Jj8ZV;nQ
zG+&Kkg_?SC!Uuj7jsq`CJhsDiOl+aX$-}}#0af8O6mgpUu2tACM-VdZP*rW{XZH$g
zCyavF9|T1!8D+y!=0yomBm&=EJ|NMxyZVZo7)YI-N`TIiCY?caq!T#OlSqgeORmdJ
zwXU}1rc!c4b&haD!<=)o0^1*d`;was)Er0?&@wTYGBLr4d|%-acv7ZnF6HKdGO;qP
zujdw1nFgY2xtZ6PepSjO&{WIyT?MQexPv<BU1m;<y;Hf#l+QrY-YL{FL8G-Qu^O|V
zDs(Q|QmIoZb>^k@N@*RX*0ctKb=CmEdfx`YO=hx|AlPB;Il-TU;QF8B{U#W;1;b8m
z3x*wbqQbEK*k-2CBb?0{?rhz9PP)k3l+Yx*EdqH6i}&Ku;BLSHcs<+&Fo74wEO@l(
z+C3gXa0KVdAcg*Fqd3f5do(1mrE*9UjQ>I4d6A!H{4MHzg_5@^`5uz-`Q@w+g9OIl
z4S7Ovm83TcWyp#esnI3}Z+sO!xGMsqg$Dc-J>gv>Li~VZ;Sl4c7Z9ZL(^TjClpIp-
z@HHwRZuonYyidsoNZfapJqTt~)S@IAdOMN7TR4>|D`tQ%j7+c(`w9j_GPz}>U&PbS
z;7I9YM!KbFvZh$jM<Tr=YqFu(%1Igj(gNBu<)kJ(vg4OOdcfQQ{*8<A-%1P6OA9cN
zE4g+6kGBeLb8|zwb9rLH4%DdzJFs8}wM?IacxbGhTe;2Tl2|)8U;_$MNpJVgLS?Of
zZEBOw9w@otS@RNj_E|~hOzT3GVGFq10Nt<-<_M;ntUhO&RJpbPPY~@0qMNxRh;Fh*
zg=puo!)!q`&gMinw~AUh@gcj#&*Hsb$QKIPE98qa%I7d{R%raeF&cN_Wy_Ap?ad^&
z&Cj7j-bGT>U<GWZl>*1sLW@HZ`FW~u_L68ch*S8BjFW>HMt(1NHIVgXKo)CDSrnp9
znmhxxvoI85)rRvR=?_`TFVT2b7Apza{}H|4GB&T^#wL!Grt~G7O_j*Bk*=s6xpP#L
zsl}^_{1$h|fBp_cUO7%AX>iHjGAhB~jLK3)4Q2@9>^=yDih^qp3WDFr;5Yv#gJOmc
zMRmsEOa^~~_vSyQgaon5;3uzRFdf}Z5nze<&Py{t8<pBvL6Uz$-3l1jDvP_M=>nS-
z2tTDRE>rRuB?}_Clu7<M)f7tk!RsGOy7yD6OaQt+d}HOAb-fD7;>;U5AfWHzNhw7K
zN0B8lT~X^wM`<hbAmV4J_39v^z}@j*5Hx5A8uSuel28gJ%v|Ga9zZ|>7Xt_pI3S1+
zfdk=b=GLx+u)}4>L__ePi{L@0Frshv;r>fdP0!5isRQ9OSq&JQ{W<~`8ylvXa5ff%
zv$_BGc_`2*LxIK-;iOQ&I2H;tw{CIrJ%`i^@!XuT`Z3*^D`!PG&mDtS2owVT2hbZ4
z43?m_4z+W}Xy<s%MJ^X;B=`0|IPZ_B`UOe^oNpf;lz2MZaZfERegk^5kl<)b!tM8X
zT1q~GaD(S2-wY2<h;`t_$#-|;x>AW;ucGb|m89r*?zc}NIvfAvJE-Iz0i2qPT?;V8
z{)hZGQw~&ka0Ah3t&C)CfCxV>yf^KL;JtIZ57-cYTEOuvt7SGy;9FOvCvqrHoWuTd
zNAxEF!PoS+_x0Y{tTAm0UyleEI@<!?*vYe|@bxNd&3!$^uWiJy2+=2X;qBGDF1)?U
z+7)kKe~k75TAa<j{fVu@PJ=YXmIIEhsK8rFE~Za<PV3>e=_3Gy6Vcb*06Vdut8ltI
z`&b}u90FdI<&TdMPO^i(QVNp8*gH-^_e6ztA<81o6v}+M(mGvizQBK>r}upLZPMdH
zK`7f4j()y-tLU~6Y76zcjb(9zd-}Ms3B(cVzl<8*tBh=MU@3(V&f!$2{O2?#q4%$y
z3Qp=>=zF7p?=OujuHpV)aikO^l_Cv$A-jQ<z-n;FbaqmyQ;uDq_)<Xax-GXh`|5yA
zi}*SsCa^Gue$4!Iwg^Hr=z|Z}D4{P5VoS*>V&)X2a1wq_hRCVc%KlUAM4KoptgC~B
djiTTe#7s)AyDrADqF9dAvCdc@T5HzYe*wY?o9zGq

literal 3738
zcmb`K&u`qu6~{^L^2c(2$&xBtPHV)D+f+%Ex+&@yuA8QnEFg!-fbGC2vo1^Tj3lnP
zBt4u>ybG=`?$tohQ;(~ouKf%8cktRva_OmuLWiQ?H%qV9^2%v}QqUW6_&6NS`+VP<
z4Vq0?!x8=AsOzn1+TZD7{xERyDgLt#4AYn%X?@+(bvid9!!uM{@v3Mmk=eIAt8aUD
z-}FrQRGIlF&2yN=Y_v7zuo_yIxvY-1&Kj(Vw!w^h+D7X;yg}R03v)|^S-QJXDXJ`x
zyr{{r&l!(oP&m{<mtCEl%cFvpzG4gi*&fX3J*hp{#(J(#bfzC-wm36#%y?act*mQ@
z2K7DA?rBv`R)&AnwJ~}|mt-|Jp2J^OS(Vxc+VAnDW8;~@%zNe8HSHSCFk@@iv(jj=
zAI60nG9JrtFXW=AJm$mEr7w5B*xkDS*_~bg_T9U8?mpb?CUMNW((lsO?QEEWmezAJ
zE`5o|yhbpP2Q)}mee~_8U7pG?i3>9oVJxv67KYdh_t8cn91EQvX`+cyjvse#cD~5C
z$U4KIKRW2#iTSn&zv7*3o;{XH+S&T-_Kv?D$l&uJ4)%E;k9V+^!Dj#DT1~9OpGpyQ
zWk+54<(0oTh`ThI57S}MeC10lTJ&WYO*eE)clA}+mhRwBw9(tG(!KI<(9R<D7hH@w
zxsg}SYvXFJ_q2(gYlr&Slt!<Tn@nfM3uA1_>cl+Mq{S+%`a-{;<rXt<!S?LDN@NjL
zMAQoladTKBuRKRAA87G)#I`p4Gh%D2*gCnbV(YNxTx{)UHgi>MaW;!>Yxh{+JkpCw
z#?$WzHk<>;VOBJQH1xYc6mjMU@>tK0YXiBrd1Es}XsS{S__2Ov=g1VaD22o0Zo<MC
zQ9IxP10J;}gGBP~K_Ch<j2VA=yz=FC@^u^~0egEZ>8BBw{Nd>R*8qybO!{B*M_DMj
zpQKy_G7*{I&A!@j3%k7Gtu1ydmiBp!N1|CMi`8D1#D1KJeh`Ht?#skydxFGNm@>h)
z3JP6ep&#!MHCzY^;D%GUFQ*YQQ75Nr8pwmf*$X2c2Yp_&lYvYJ(vK3*3~s!)7EjBk
zIe6=5+{APm^j2bgYFhNNi)XlFmgjIkLmpE!I(kEIRo09(@K_T09`3wCB+Y(C<VOhf
z=$|i%l;H7D8-v5K#f-5H?)1loNI|MGspML29~xsvR(oddC^n68Ew}R;(J8f_mDh4}
z-vGrHC^nfrwsI%iqxQ+g+-And;PS~uc-mmyS=Wf-7Y3cH=c%_o*IS?UHs*R8=yfJF
z=)+|OYbt%1tOb3Ty*l*KW=rfGTZT$jls+2hSM<vUTb(K829(nJMjw6vt++}njoejQ
zX|Ny6X~li!GFxc{XEUv=?cNtFSVyr$hC<-ANhB#UjUNyCkGOz(8sR=z<Gx~k$8)=a
zBhitD3ZtJP?^g$LC_Q(OQj(|sQ5}g0S!^>3GAUMRa0;zBPv!!dbut&pyhY}1n9;^!
z;iNo)viwve1nE>Z?ZQY|QKP)LBY>^Ai~+n=h1V&6afM#+4vb2}pt`7&V9I)=WFxMU
z4~a(nm<%bVu$UaCJew+~E`CBc2s-gDnV*u`fbq^RdMgC!)uJRx{k<sIFY0qET8ie>
zG?ktTTk?KNVqT=43kzQa?a;?(AHh&2NB%addev}B?JVoK(k;V<n((Jo<?1bCX`1P;
z<EgGm&z1+hd8Vs><6`u;QdefFD+}ZsAQV|%sRw(7btodYx3!1Y#|~6hnN*=N2P&&c
za{@h~Hn_QyyQRYH+|4a03^jo2#FGzD6P#Xc;!;I8G;+&#W<BVe_Q?9A0kt$4b{}Om
zX+kYluQjbWtcBS?HA*dQcJ398EPbO5{{@ZIl}6fmU1_Avmgh85e^zI0r4gLXG_tZg
zhb<9%R8a*&{1E5j5=>DmV;Qph;(h8RITgBJ=pRnyb)o>`erJyPtFJM?k8MRc9@TkO
zCC-VT(XiKOSO@B3Dz7W%+o$|ba2G!(ce|TJgMOT$9!gQzf~Qf?<!4eqrM<C;_Toop
z@rQgN52j*RJ~bV%T)3!sRIoypi5r-cw|d%%YKizIec~eSS5CVF`P*>LNVPAKU#k%C
zF6?S$Szn$OkC%}D7Jg?#{tw(8{q=u<{O0SBQ_(1KUIaR@o`OD0`LyksZi!F`0CBz7
zm^4|FSqk?0e}Oe^l9q<w{(qnrA7c?zbWUnP39~8?Cj{Rl$I}@9;!PN5gGU)QrD{*?
zXEKn3X%e5KTKtMeRX|^wi}r#iPw{;k<Xbe(ugQEuW|qz0z_s`dd8$-?`OWX9#QZ5a
zD~jJuzt>wO^|Qgg@;w89bNW$~Q3ZRIbJ`TGg7Iaf^p;U>k$wlSjiy(d{s2H7S3gwL
zf)!R8#4O0Bd4$sabm3IVqs_jc{gt3%AV@5N#+uu`sePQ<!a2A42^&QG6CEBICAx+x
PB&&u~cb1&1&gK6Ad^?%1

diff --git a/crisp/extract/crisp_extract_function.py b/crisp/extract/crisp_extract_function.py
index 2d075ce..f36b83f 100644
--- a/crisp/extract/crisp_extract_function.py
+++ b/crisp/extract/crisp_extract_function.py
@@ -14,23 +14,25 @@ def connect_crisp(config):
         print("[FUNCTION_ERROR]-connect_crisp")
         return False
     
-def list_website_operators_csv(website_id, client, path, filename):
+def list_website_operators_json(website_id, client, path, filename):
     try:
         output_location = path + filename
         list_website_operators_json = client.website.list_website_operators(website_id)
         list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json, sep='__')
 
         # Datetime add
-        list_website_operators_df['api_called_at'] = date.today() - timedelta(days = 1)
+        list_website_operators_df['api_called_at'] = datetime.now()
 
-        list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        # list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        list_website_operators_df.to_json(output_location, orient='records', force_ascii = False)
         print("[Download Complete]",filename)
+
     except Exception as e:
         print(e)
-        print("[FUNCTION_ERROR]-list_website_operators_csv")
+        print("[FUNCTION_ERROR]-list_website_operators_json")
         return False         
 
-def list_people_profiles_csv(website_id, client, path, filename):
+def list_people_profiles_json(website_id, client, path, filename):
     try:
         output_location = path + filename
         list_people_profiles_df = pd.DataFrame()
@@ -53,23 +55,24 @@ def list_people_profiles_csv(website_id, client, path, filename):
         list_people_profiles_df['active__last']=(pd.to_datetime(list_people_profiles_df['active__last'],unit='ms')) 
 
         # Datetime add
-        list_people_profiles_df['api_called_at'] = date.today() - timedelta(days = 1)
+        list_people_profiles_df['api_called_at'] = datetime.now()
 
-        list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        # list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        list_people_profiles_df.to_json(output_location, orient='records', force_ascii = False)
         print("[Download Complete]",filename)
 
     except Exception as e:
         print(e)
-        print("[FUNCTION_ERROR]-list_people_profiles_csv")
+        print("[FUNCTION_ERROR]-list_people_profiles_json")
         return False         
 
 
-def get_people_data_csv(website_id, client, path, filename, list_people_profiles_csv_path):
+def get_people_data_json(website_id, client, path, filename, list_people_profiles_json_path):
     try:
         output_location = path + filename
         get_people_data_df = pd.DataFrame()
 
-        person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
+        person_ids = pd.read_json(list_people_profiles_json_path)['people_id']
 
         # Get marking by CustomerService Operator
         for people_id in person_ids:
@@ -85,40 +88,104 @@ def get_people_data_csv(website_id, client, path, filename, list_people_profiles
         get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "__")
 
         # Datetime add
-        get_people_data_df['api_called_at'] = date.today() - timedelta(days = 1)
+        get_people_data_df['api_called_at'] = datetime.now()
 
-        get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        # get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        get_people_data_df.to_json(output_location, orient='records', force_ascii = False)
         print("[Download Complete]",filename)
 
     except Exception as e:
         print(e)
-        print("[FUNCTION_ERROR]-get_people_data_csv")
+        print("[FUNCTION_ERROR]-get_people_data_json")
         return False         
 
 
 
-def get_people_subscription_status_csv(website_id, client, path, filename, list_people_profiles_csv_path):
+def get_people_subscription_status_json(website_id, client, path, filename, list_people_profiles_json_path):
     try:
         output_location = path + filename
         get_people_subscription_status_df = pd.DataFrame()
 
-        person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
+        person_ids = pd.read_json(list_people_profiles_json_path)['people_id']
         
         # Get email subscription status
         for people_id in person_ids:
-            pget_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
-            temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json, sep='__')
+            get_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
+            temp_df = general_function.json_normalize_to_df(get_people_subscription_status_df_json, sep='__')
             temp_df['people_id'] = people_id
             get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True)
 
         # Datetime add
-        get_people_subscription_status_df['api_called_at'] = date.today() - timedelta(days = 1)
+        get_people_subscription_status_df['api_called_at'] = datetime.now()
 
         get_people_subscription_status_df.columns = get_people_subscription_status_df.columns.str.replace(".", "-")
-        get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        # get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
+        get_people_subscription_status_df.to_json(output_location, orient='records', force_ascii = False)
+
         print("[Download Complete]",filename)
 
     except Exception as e:
         print(e)
-        print("[FUNCTION_ERROR]-get_people_subscription_status_csv")
-        return False         
\ No newline at end of file
+        print("[FUNCTION_ERROR]-get_people_subscription_status_json")
+        return False         
+
+
+
+def list_conversations_json(website_id, client, path, filename):
+    try:
+        loop_flag = True
+        page_number = 1
+        output_location = path + filename
+        conversations_list_df = pd.DataFrame()
+
+        while loop_flag:
+            conversations_list_df_json = client.website.list_conversations(website_id, page_number)
+
+            temp_df = general_function.json_normalize_to_df(conversations_list_df_json, sep='__')
+            conversations_list_df = pd.concat([conversations_list_df, temp_df], ignore_index=True)
+
+            if conversations_list_df_json == []:
+                loop_flag = False
+            else:
+                page_number += 1
+        
+        conversations_list_df.columns = conversations_list_df.columns.str.replace("-", "_")
+        conversations_list_df['api_called_at'] = datetime.now()
+
+        conversations_list_df.to_json(output_location, orient='records', force_ascii = False)
+        print("[Download Complete]",filename)
+        
+    except Exception as e:
+        print(e)
+        print("[FUNCTION_ERROR]-list_conversations_json")
+        return False  
+
+
+def get_message_in_conversations_json(website_id, client, path, filename, list_conversations_json_path):
+    try:
+        output_location = path + filename
+        get_message_in_conversations_df = pd.DataFrame()
+        session_ids = pd.read_json(list_conversations_json_path)['session_id']
+
+        for session_id in session_ids:
+            print(session_id) #NEED DELETE
+
+            get_message_in_conversations_json = client.website.get_messages_in_conversation(website_id, session_id, "")   
+            temp_df = general_function.json_normalize_to_df(get_message_in_conversations_json, sep='__')
+            get_message_in_conversations_df = pd.concat([temp_df, get_message_in_conversations_df], ignore_index=True)
+
+            while (temp_df.shape[0] == 40):       #Maximum can get 40 conversation per request, Request for date before if the temp_df is full
+                query = { "timestamp_before" : str(temp_df.at[0,'timestamp']) }
+                get_message_in_conversations_json = client.website.get_messages_in_conversation(website_id, session_id, query)
+                temp_df = pd.json_normalize(get_message_in_conversations_json, sep='__')
+                get_message_in_conversations_df = pd.concat([temp_df,get_message_in_conversations_df], ignore_index=True)
+            
+        get_message_in_conversations_df['api_called_at'] = datetime.now()
+        get_message_in_conversations_df.to_json(output_location, orient='records', force_ascii = False)
+        print("[Download Complete]",filename)
+        
+
+    except Exception as e:
+        print(e)
+        print("[FUNCTION_ERROR]-get_message_in_conversations_json")
+        return False 
\ No newline at end of file
diff --git a/crisp/load/__pycache__/crisp_load_function.cpython-39.pyc b/crisp/load/__pycache__/crisp_load_function.cpython-39.pyc
index 9dc5c9793b28a0a262c1c6eaf86a5841cf4c2070..ce65279483e1f67f0f085d2a07b2a8b71cb9ccc6 100644
GIT binary patch
delta 20
acmX@hewLj(k(ZZ?0SKPY{;-kz5HkQe!v-t>

delta 20
acmX@hewLj(k(ZZ?0SLDH^>5@p#0&s6zy(eK

diff --git a/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc b/exchange/extract/__pycache__/exchange_extract_function.cpython-39.pyc
index c06031b5388fefe7f8a9336ff0aae0d8174244d8..f22ae4ad9474101df9ce2708f0bea8159408695a 100644
GIT binary patch
delta 307
zcmey#`;(VDk(ZZ?0SKPY{;-ido{{m^<ZMP6(OYcE`FUxX=}|1Xl_^QL7*kg=6!A`;
z!?>K0f3pkIZboTdpcYN0TTFTex0sXjbBaKG2*Ed5iltQi7H3LoPG)XqNovt8&cf3C
zlGNmk#G+f1m$Ag^fb6@)3Y5w%zQvwek(yjul6s3HEwu!wBqyhc4J6H$mS2>cSaNH!
zKC7|GE#AOT{}9)BUr#?zUq_!?oaIFzgY(OaZ?Pul<QJzFae|z~AH@+I=o9ba=^Rod
zH+dRs0MIkfSv7#90Gp@)58Q=dxAILkWeedj0_o!@(wsbzO;cVLq=FwrfP|5}1l4ep
GO#uM6SXVj#

delta 307
zcmey#`;(VDk(ZZ?0SF%bvDnBR&&U`xIh#>NG>R=bKQApaJ&Gl_G9~F2W9mwVBJRm^
z7?(5hZgyeX%_z+c)S}6Bi%HKQia9wyrwGJ{5ImEmSW3mCI8#z{GIKLaQj4NE3rq7$
zQj;?hi=rkkV~N$_1ggEo3Y5w%j$%)(NKGy+NsZ!2ODzE^$;l~V14*-`<rn29mPAd~
zXEip7;tdS-4{?q6_4M=fb@YkiEH45XoL^pii#0hXzc{ss1LPe3D30JjpLiEf=a3?a
z$<tT^fS!5IssSVg*hB?DzC>}RDO(7KDo7tkk;3GOY?|`oAQikI0wj#27VJ5m$v4>)
E0I&s64*&oF

diff --git a/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc b/exchange/load/__pycache__/exchange_load_function.cpython-39.pyc
index 5aebbb1bdb19b47013547e82dfcc7985b74403bc..359d67671c30e4c276e7e4673304fb24f02e1289 100644
GIT binary patch
delta 20
acmcb~ev_R$k(ZZ?0SKPY{;-kz95VnsqXtj_

delta 20
acmcb~ev_R$k(ZZ?0SHd*HQ&g6ju`+s2L+q}

diff --git a/main.py b/main.py
index 9f2fff4..b977aaa 100644
--- a/main.py
+++ b/main.py
@@ -1,5 +1,6 @@
 import os
 from os import walk
+import pandas as pd
 import general_function
 import exchange.extract.exchange_extract_function as exchange_extract_func
 import exchange.load.exchange_load_function as exchange_load_func
@@ -92,16 +93,22 @@ def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
     CLIENT = crisp_extract_func.connect_crisp(CRISP_API_CONFIG)
 
     #Get operators csv
-    crisp_extract_func.list_website_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_website_operators.csv')
+    crisp_extract_func.list_website_operators_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_website_operators.json')
 
     #Get users(people) csv
-    crisp_extract_func.list_people_profiles_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_people_profiles.csv')  
+    crisp_extract_func.list_people_profiles_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_people_profiles.json')  
 
-    # Get peoples manually marking by operator
-    crisp_extract_func.get_people_data_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv')
+    #Get peoples manually marking by operator
+    crisp_extract_func.get_people_data_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_data.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.json')
 
-    # Get peoples subscription status
-    crisp_extract_func.get_people_subscription_status_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.csv', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.csv')
+    #Get peoples subscription status
+    crisp_extract_func.get_people_subscription_status_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_people_subscription_status.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_people_profiles.json')
+
+    #Get current converstation list
+    crisp_extract_func.list_conversations_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_list_conversations.json')
+
+    #Get message in conversations
+    crisp_extract_func.get_message_in_conversations_json(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_get_message_in_conversations.json', './crisp/extract/crisp_temp_extract_csv/crisp_list_conversations.json')
 
 def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
     #Config file name
@@ -168,4 +175,4 @@ def main():
     print("------------------------------------------------------------------------------------------------")
 
 if True:
-    main()
+    main()
\ No newline at end of file
diff --git a/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc b/shipping/extract/__pycache__/shipping_extract_function.cpython-39.pyc
index bb268101033c3bcf00d9372a520b17d62096bb14..b2d8853c432b68c921c2e4d9d326da2a934c2940 100644
GIT binary patch
delta 20
acmey#`;(VDk(ZZ?0SKPY{;-kz9UA~beg?k)

delta 20
acmey#`;(VDk(ZZ?0SJ6w7i{Ey#|8jBs0H-^

diff --git a/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc b/shipping/load/__pycache__/shipping_load_function.cpython-39.pyc
index b895fda1135b4bfcb139f227f69fd34150e8725a..01aa2be379b46392b5eba73b294e14e61e426a5f 100644
GIT binary patch
delta 20
acmcb~ev_R$k(ZZ?0SKPY{;-kz95VnsqXtj_

delta 20
acmcb~ev_R$k(ZZ?0SF3T7i{D{#|!{C$pxwa