2) removedup.py

# to remove duplicates in the data, based on sensing timestamp

import os
import io

if not os.path.exists("android_remdup"):
    os.makedirs("android_remdup")
files= os.listdir("android") 
s = []
for file in files:
    alldata =dict()
    for l in open("android\\"+file).readlines():
        ll = l.split(',')
        if ll[3] not in alldata.keys():
            alldata[ll[3]]=l
    ow = open("android_remdup\\"+file,'a+')
    for v in alldata.values():
        ow.write(v)
    ow.close()

if not os.path.exists("ios_remdup"):
    os.makedirs("ios_remdup")
files= os.listdir("ios") 
s = []
for file in files:
    alldata =dict()
    for l in io.open("ios\\"+file, encoding='utf', errors='ignore').readlines():
        ll = l.split(',')
        if ll[3] not in alldata.keys():
            alldata[ll[3]]=l
    ow = open("ios_remdup\\"+file,'a+')
    for v in alldata.values():
        ow.write(v)
    ow.close()

firstline='Userid,BSSID:,State,senseStartTimeMillis:\n'
title = dict()
if not os.path.exists("android_wifiraw_remdup"):
    os.makedirs("android_wifiraw_remdup")
files= os.listdir("android_wifiraw") 
s = []
for file in files:
    alldata =dict()
    for l in open("android_wifiraw\\"+file).readlines():
        ll = l.split(',')
        if ll[3] not in alldata.keys():
            alldata[ll[3]]=l
    ow = open("android_wifiraw_remdup\\"+file,'a+')

    for v in alldata.values():
        ow.write(v)
    ow.close()

title = dict()
if not os.path.exists("ios_wifiraw_remdup"):
    os.makedirs("ios_wifiraw_remdup")
files= os.listdir("ios_wifiraw") 
s = []
for file in files:
    alldata =dict()
    for l in io.open("ios_wifiraw\\"+file, encoding='utf', errors='ignore').readlines():
        ll = l.split(',')
        if ll[3] not in alldata.keys():
            alldata[ll[3]]=l
    ow = open("ios_wifiraw_remdup\\"+file,'a+')

    for v in alldata.values():
        ow.write(v)
    ow.close()