diff --git a/config/EXCHANGE_EXTRACT_SQL.json b/config/EXCHANGE_EXTRACT_SQL.json index 2dee529..3889db1 100644 --- a/config/EXCHANGE_EXTRACT_SQL.json +++ b/config/EXCHANGE_EXTRACT_SQL.json @@ -12,7 +12,7 @@ "segment_companies" : "SELECT *, now() AS '_source_loaded_at' FROM segment_companies;", "service_types" : "SELECT *, now() AS '_source_loaded_at' FROM service_types;", "transactions" : "SELECT *, now() AS '_source_loaded_at' FROM transactions;", - "transaction_detail" : "SELECT *, now() AS '_source_loaded_at' FROM transaction_detail;", + "transaction_detail" : "SELECT id, transaction_id, product_code, REPLACE(product_name, '\n', '') AS product_name, quantity, price, amount, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM `transaction_detail`;", "users" : "SELECT id, CONCAT(SUBSTR(name,1,2),SUBSTR(name,5,1),REPEAT('',LOCATE('@',name)-3),SUBSTR(name,LOCATE('@',name)-1,3),REPEAT('',LENGTH(name)-LOCATE('.',REVERSE(name))-LOCATE('@',name)-1),SUBSTR(name,LENGTH(name)-LOCATE('.',REVERSE(name)))) AS name, CONCAT(SUBSTR(email,1,2),SUBSTR(email,5,1),REPEAT('',LOCATE('@',email)-3),SUBSTR(email,LOCATE('@',email)-1,3),REPEAT('',LENGTH(email)-LOCATE('.',REVERSE(email))-LOCATE('@',email)-1),SUBSTR(email,LENGTH(email)-LOCATE('.',REVERSE(email)))) AS email, type, status, deleted_at, created_at, updated_at, now() AS '_source_loaded_at' FROM users;", "user_email_verifications" : "SELECT id, CONCAT(SUBSTR(email,1,2),SUBSTR(email,5,1),REPEAT('',LOCATE('@',email)-3),SUBSTR(email,LOCATE('@',email)-1,3),REPEAT('',LENGTH(email)-LOCATE('.',REVERSE(email))-LOCATE('@',email)-1),SUBSTR(email,LENGTH(email)-LOCATE('.',REVERSE(email)))) AS email, is_complete, is_active, is_sent, created_at, updated_at, now() AS '_source_loaded_at' FROM user_email_verifications;", "wallets" : "SELECT *, now() AS '_source_loaded_at' FROM wallets;" diff --git a/exchange/extract/exchange_extract_function.py b/exchange/extract/exchange_extract_function.py index 9fd7d05..2e250c6 100644 --- a/exchange/extract/exchange_extract_function.py +++ b/exchange/extract/exchange_extract_function.py @@ -37,7 +37,7 @@ def mysql_dict_sql_to_csv(SQL_DICT, path, mysqldb, mysqlcursor): filename = '/{}_{}.csv'.format(mysqldbname, tablename) headers = [col[0] for col in mysqlcursor.description] # get headers mysqlresult.insert(0, tuple(headers)) - fp = open(concat(path,filename), 'w', newline = '\r\n', encoding = 'utf-8-sig') + fp = open(concat(path,filename), 'w', newline = '', encoding = 'utf-8-sig') myFile = csv.writer(fp,delimiter='|',quotechar='"', quoting=csv.QUOTE_MINIMAL) myFile.writerows(mysqlresult) fp.close()