cief-dashboard/apps/upload_data.py

from app import *
import base64
import io
import pgeocode

def load_data(contents):
    content_type, content_string = contents.split(',')
    decoded = base64.b64decode(content_string)
    try:
        if 'csv' in content_type:
            # Assume that the user uploaded a CSV file
            df = pd.read_csv(io.StringIO(decoded.decode('utf-8')))
            df.to_csv('data/test0.csv')
            #return(html.H1('File loaded successfully'))
            return(dbc.Alert("This is a success alert! Well done!", color="success"))
        elif 'openxml' in content_type:
            # Assume that the user uploaded an excel file
            df = pd.read_excel(io.BytesIO(decoded),sheet_name='Monthly',header=1,parse_dates=['进仓 日期 (DATE)','ETA\nEstimated arrived Port 预计到港   日 期','DATE OF DELIVERY ','ACTUAL ETA ','Unstuffing Date'])
            df.to_csv('data/warehouse0.csv', index=False)
            #return(html.H1('File loaded successfully'))
            return(dbc.Alert("File loaded successfully", color="success"))
    except Exception as e:
        print(e)
        return('There was an error processing this file.')

def update_data():
    df=pd.read_csv('./data/warehouse0.csv')

    df=df.iloc[:,[0,1,6,7,8,10,16,18,20]]
    df.columns=['Date','Destination port','Marking','Description','CTN','Postcode','Delivery status','Remark','CBM']
    #df=df.iloc[:,[0,1,5,6,7,8,10,11,15,16,17,18,20,22]]
    #df.columns=['Date','Destination port','ETA','Marking','Description','CTN','Postcode','Delivery date','Unstuffing date','Delivery status','Release date','Remark','CBM','Real ETA']

    # select postcode
    df.loc[:,'Postcode']=df.loc[:,'Postcode'].str.extract("\D(\d{5})\D",expand=False)
    df.loc[:,'Postcode'].fillna('Unkown', inplace=True)

    # convert date format
    df.loc[:,'Date']=pd.to_datetime(df.loc[:,'Date'],errors='coerce')

    # correct ctn number
    df.loc[:,'CTN']=df.loc[:,'CTN'].astype('str').str.extract("(\d+)",expand=False)
    df.loc[:,'CTN']=pd.to_numeric(df.loc[:,'CTN'],errors='coerce',downcast='integer')
    df.loc[:,'CTN'].fillna(0, inplace=True)
    df.loc[:,'CTN']=df.loc[:,'CTN'].astype('int')


    df.loc[:,'Cleaned Marking']=df.loc[:,'Marking']
    # correct string errors
    convertion={"[Cc][Ii][Ee][Ff]":"CIEF/","[Cc][Ee][Ii][Ff]":"CIEF/","[Cc][Ii][Ee]/":"CIEF/","\n":"/","\(.*\)":"/", "-":"/"," ":"/", "\.":"/","/+":"/","$":"/"}
    df.loc[:,'Cleaned Marking'].replace(to_replace=convertion,regex=True,inplace=True)
    # extract markeing
    df.loc[:,'Cleaned Marking']=df.loc[:,'Cleaned Marking'].str.extract("(CIEF/\w*)/",expand=False)

    # clean remark
    remark_convertion={"^.*HOLD \(PIA\)":"Hold (PIA)", "^.*[Ss][Ee][Ll][Ff].*$":"Self collection", "^.*[Dd][Ee][Ly][Aa][Yy].*$":"Delay", "^.*[Cc][Uu][Ss][Tt][Oo][Mm].*$|^.*[Cc][Hh][Ee][Cc][Kk].*$":"Custom Check", "^.*IN.*$|^.*[Dd][Aa][Yy].*$|^.*[Uu][Rr][Gg][Ee][Nn][Tt].*$":"Urgent delivery"}
    df.loc[:,'Remark'].replace(to_replace=remark_convertion,regex=True,inplace=True)
    df.loc[:,'Remark']=df.loc[:,'Remark'].str.extract("(Urgent delivery|Custom Check|Self collection|Hold \(PIA\)|Delay)",expand=False)
    df.loc[:,'Remark'].fillna('No remark', inplace=True)

    # dropna
    df.dropna(subset=['Cleaned Marking'],inplace=True)
    # limit digits after the float point

    df.to_csv('data/warehouse1.csv',index=False)

def get_derivative_data():
    df=pd.read_csv('data/warehouse1.csv',parse_dates=['Date'])
    df.loc[:,'Postcode']=df.loc[:,'Postcode'].astype('str').str.pad(width=5,side='left',fillchar='0')

    daily_cubic=df.loc[:,'CBM'].groupby(by=df.loc[:,'Date'].dt.to_period("D")).sum()
    daily_cubic=pd.DataFrame(daily_cubic)
    daily_cubic.columns=['Daily CBM']
    daily_cubic.reset_index(inplace=True)
    daily_cubic.loc[:,'Date']=daily_cubic.loc[:,'Date'].apply(pd.Period.to_timestamp)
    df=pd.merge(df,daily_cubic,how="left")

    daily_ctn=df.loc[:,'CTN'].groupby(by=df.loc[:,'Date'].dt.to_period("D")).sum()
    daily_ctn=pd.DataFrame(daily_ctn)
    daily_ctn.columns=['Daily CTN']
    daily_ctn.reset_index(inplace=True)
    daily_ctn.loc[:,'Date']=daily_ctn.loc[:,'Date'].apply(pd.Period.to_timestamp)
    df=pd.merge(df,daily_ctn,how="left")

    nominating = pgeocode.Nominatim('my')
    postcodes=df.loc[:,'Postcode'].unique()
    location_df=nominating.query_postal_code(postcodes)
    location_df=location_df.iloc[:,[0,2,3,9,10]]
    location_df.columns=['Postcode','Place name','State name','Latitude','Longitude']
    location_df.loc[:,'State name'].fillna(value='Other', inplace=True)
    df=pd.merge(df,location_df, how='left',on='Postcode')

    df=df.round(2)

    df.to_csv('data/warehouse2.csv',index=False)

layout = html.Div([
    html.H1('Upload warehouse summery data'),

    dcc.Upload(
        id='input-excel',
        children=html.A('Drag or Select Files'),
        style={
            'width': '100%',
            'height': '60px',
            'lineHeight': '60px',
            'borderWidth': '1px',
            'borderStyle': 'dashed',
            'borderRadius': '5px',
            'textAlign': 'center',
            'margin': '10px'}
        ),

    html.Div(id='output-upload_result'),
    ])

@app.callback(Output('output-upload_result', 'children'),
              [Input('input-excel', 'contents')])
def update_output(data):
    if data is not None:
        children = load_data(data)
        update_data()
        get_derivative_data()
        return(children)