scr_002_process-csvs-into-pickles.py

import pandas as pd
import numpy as np
import time

# List of all filenames
od_fns = ["./DataForUConn/201407OD/201407.gz/part-m-{x}".format(x=str(x).rjust(5,"0")) for x in range(12)]
gps_fns = ["./DataForUConn/201407GPS/part-r-{x}".format(x=str(x).rjust(5,"0")) for x in range(2)]

od_headers = ["id", "ptime", "dtime", "plon", "plat", "dlon", "dlat"]
gps_headers = ["id", "color", "lon", "lat", "time", "speed", "noMeaning"]

# Time formatter function; string --> unix timestamp.
# (Timezone data is lost, but shouldn't matter, assuming each string comes from the same timezone.)
formattime = lambda timestr: int(time.mktime(time.strptime(timestr, "%Y-%m-%dT%H:%M:%S.000Z")))

for od_id, od_fn in enumerate(od_fns):
    print(f"Opening OD {od_id}...")
    with open(od_fn) as f:
        odlines = [line.split(",") for line in f.read().strip().split("\n")]
        
    print(f"Processing OD {od_id}...")
    oddf = pd.DataFrame(odlines, columns=od_headers)
    oddf["ptime"] = oddf["ptime"].apply(formattime)
    oddf["dtime"] = oddf["dtime"].apply(formattime)
    oddf["plon"] = oddf["plon"].astype(np.float32)
    oddf["plat"] = oddf["plat"].astype(np.float32)
    oddf["dlon"] = oddf["dlon"].astype(np.float32)
    oddf["dlat"] = oddf["dlat"].astype(np.float32)
    
    print(f"Saving OD {od_id} pickle...")
    oddf.to_pickle("./data/od_part-m-{x}.pkl".format(x=str(od_id).rjust(3,"0")),
                    compression=None)


for gps_id, gps_fn in enumerate(gps_fns):
    print(f"Opening GPS {gps_id}...")
    with open(gps_fn) as f:
        gpslines = [line.split(",") for line in f.read().strip().split("\n")]
    
    print(f"Processing GPS {gps_id}...")
    gpsdf = pd.DataFrame(gpslines, columns=gps_headers)
    gpsdf["time"]  = gpsdf["time"].apply(formattime)
    gpsdf["lon"]   = gpsdf["lon"].astype(np.float32)
    gpsdf["lat"]   = gpsdf["lat"].astype(np.float32)
    gpsdf["speed"] = gpsdf["speed"].astype(int)
    
    print(f"Saving GPS {gps_id} pickle...")
    gpsdf.to_pickle("./data/gps_part-r-{x}.pkl".format(x=str(gps_id).rjust(3,"0")),
                    compression=None)