import pandas as pd
from glob import glob

#data = sorted(glob('/home/disk/picea/mauger/2024_03_TNC_StormwaterHeatmap/DATA/SeaTac_ds3505_Hourly/72*.csv'))
#site = ['Blnghm', 'Bremerton', 'OlyArpt', 'PortAngeles', 'Quillayute', 'Quinault']
site = ['FridayHarbor', 'OrcasArpt', 'PortAngeles']

def process(ifile):
    df = pd.read_csv(ifile, low_memory=False)
    times = pd.to_datetime(df.DATE, format='%Y-%m-%dT%H:%M:%S')
    cf = df.AA1.str.split(',', expand=True)
    cf.columns = ['AA1', 'AA2', 'AA3', 'AA4']
    cf = cf[cf.AA1=='01']
    cf = cf.apply(pd.to_numeric, errors='coerce').dropna()
    cf = cf.astype(int)    
    cf['AA2'] = cf.AA2/10
    
    cf.insert(0, 'YYYY', times.dt.year)
    cf.insert(1, 'MM', times.dt.month)
    cf.insert(2, 'DD', times.dt.day)
    cf.insert(3, 'HH', times.dt.hour)
    cf.insert(4, 'NN', times.dt.minute)
    cf.insert(5, 'SS', times.dt.second)
    cf = cf.dropna()
    return cf

def process_site(site):
    df = pd.DataFrame()
    data = sorted(glob('/home/disk/margaret/mauger/2024_03_TNC_StormwaterHeatmap/DATA/{}_ds3505_Hourly/*.csv'.format(site)))

    for d in data:    
        print(d)
        cf = process(d)
        if(len(cf)!= 0):
            df = pd.concat([df, cf])
    print(df)
    df.to_csv('data/{}_ds3505.csv'.format(site), index=False, float_format='%0.2f')

for s in site:
    print(s)
    process_site(s)