csv2math.py

import sys
import os
#directory= "data_all/data_with_diag/"
directory = "/Users/omar/last_data/"
import math
charts = []
chart_count = 0
datasets = {}
debug=False
files1 = os.listdir(directory)
files = files1[::1]
for file in files1:
	if "TN" in file:
		files.remove(file)
		files.append(file)
broken = []
for filename in files:
	if "csv" not in filename:
		continue
	#filename = "data2/all_data_standard100K_noE_Q.csv"
	#print(filename)
	with open(directory+filename, "r") as f:
		skip = False
		skip_others = False
		tag=""
		chart = ""
		for line in f.readlines():
			#print(line)
			if "Data" in line:
				#its a header, generate label, set flag to skip next one:
				#check error default and Q default
				# print("Getting chart name")
				chart = line[line.find("iterations, ") + len("iterations, "):]
				
				if "L = 14.78"  in line:
					chart = chart[:chart.find("Pz")]
					
					special = True
					if "Global Info = True" in line:
						tag = "Global"
					elif "Smart = True" in line:
						tag = "IA"
					elif "Smart = False" in line:
						tag = "NIA"
				else:
					special = False
					tag = ""

				if "Global Info = True" in line:
					tag = "Global"
				elif "Smart = True" in line:
					tag = "IA"
				elif "Smart = False" in line:
					tag = "NIA"

				if chart in datasets:
					pass
				else:
					datasets[chart] = {}

				#check routing 
				skip = True
				# print("Skip is now")
				continue

			if skip:
				# print("Getting headers")
				# print(line)
				#check routing and create entry for each data set, i
				#if dataset already exists set flag to only do data3
				#finalize label add data sets, skip
				skip = False
				var = line[0]
				if var == "P":
					var = "L"
				# print("var: ", var) 
				d0 = var+"NoTN"+tag
				d1 = var+"CentralTN"+tag
				d2 = var+"CornerTN"+tag
				d3 = var+"LinearTN"+tag
				d4 = var+"AsymmetricTN"+tag
				d5 = var+"2Hops"+tag
				if d0 in datasets[chart]:
					skip_others = True
				else:
					skip_others = False
					datasets[chart][d0] =[]
					datasets[chart][d1] =[]
					datasets[chart][d2] =[]
				datasets[chart][d3] =[]
				datasets[chart][d4] =[]
				datasets[chart][d5] =[]
				# print(d0)
				# print(d2)
				# print(d1)
				# print(d3)
				continue
			try:
				#print(line)
				if not special:
					if not skip_others:
						datasets[chart][d0].append(str((float(line.split(",")[0].strip()),float(line.split(",")[1].strip()))))
						datasets[chart][d1].append(str((float(line.split(",")[0].strip()),float(line.split(",")[2].strip()))))
						datasets[chart][d2].append(str((float(line.split(",")[0].strip()),float(line.split(",")[3].strip()))))
						
					datasets[chart][d3].append(str((float(line.split(",")[0].strip()),float(line.split(",")[4].strip()))))
					datasets[chart][d4].append(str((float(line.split(",")[0].strip()),float(line.split(",")[5].strip()))))
					datasets[chart][d5].append(str((float(line.split(",")[0].strip()),float(line.split(",")[5].strip()))))
				if special:
					if not skip_others:
						datasets[chart][d0].append(str((float(line.split(",")[0].strip()),math.log10(2*float(line.split(",")[1].strip())))))
						
			except Exception as e:
				#del datasets[chart]
				broken.append(chart)
				pass
				#print("Exception:",  e)
				#print(line)
				#print(datasets[chart].keys())
				#if input("Skip? y/n : ") == "n":
				#	raise RuntimeError

				# print(chart)
				# print(d0)
				# print(d2)
				# print(d1)
				# print(d3)

			#readlines and add data
		
		

		print("\n(*Data from {}:*)".format(filename.strip()))
for chart in sorted(datasets.keys()):
	charts.append(chart)
	chart_count+=1
	print("\n(*Chart {}*)".format(chart[:-2]))
	print("Print[\"{}\"]".format(chart[:-2]))
	remove =  []
	for line in datasets[chart]:
		data = datasets[chart][line]
		if not len(set([y[1] for y in[ x.split(",") for x in data]])) > 1:
			remove.append(line)
	if len(remove) > 1:
		for line in remove:
			del datasets[chart][line]
	remove = []
	for line in datasets[chart]:
		data = datasets[chart][line]

		data[:] = [x.replace("e","*^") for x in data]

		if True or len(set([y[1] for y in[ x.split(",") for x in data]])) > 1:
			datastr = "{} = {{{}}};".format(line, ", ".join(data))
			datastr = datastr.replace(")","}")
			datastr = datastr.replace("(","{")
			print(datastr)
		else:
			remove.append(line)
	if len(remove) > 1:
		for line in remove:
			del datasets[chart][line]
	print("data = {{{}}}".format(",".join([x for x in datasets[chart]])))
	print("legend = {{{}}}".format(",".join(["\""+str(x)+"\"" for x in  datasets[chart]])))
	print()
	chart_size = str(len(datasets[chart]))
	print("ListLinePlot[data, PlotLegends -> legend, PlotRange -> All, AxesLabel -> {\""+line[0]+"\", \"Key Rate\"}]")

print("")
for chart in broken:
	print("(*Deleted chart {}*)".format(chart))
# 	print("""plots = Append[
#    MapThread[
#     ListPlot[#1, Joined -> True, PlotStyle -> #2] &, {datasets, 
#      ColorData[97] /@ Range["""+chart_size+"""]}], ListPlot[{}]];
# Manipulate[
#  Show[plots[[datasetNo /. {} -> 4]], 
#   PlotRange -> All], {{datasetNo, {1}, ""}, 
#   Thread[Range["""+chart_size+"""] -> 
#     MapThread[
#      Row[{Pane@
#          Graphics[{#1, Rectangle[{0, 0}, {10, 1}]}, ImageSize -> 20], 
#         Part[legend, #2]}, 
#        BaselinePosition -> Center] &, {ColorData[97] /@ Range["""+chart_size+"""], 
#       Range["""+chart_size+"""]}]], ControlType -> CheckboxBar, 
#   BaselinePosition -> Center, Method -> "Active", 
#   Appearance -> "Vertical"}, ControlPlacement -> Right]""")

# print(len(charts))
# print(len(set(charts)))
# print(chart_count)
# for chart in charts:
# 	continue
# 	if charts.count(chart)  > 1:
# 		print(charts.count(chart))
# 		print(chart)