bag_of_words.py

import pandas as pd 

df = pd.read_csv('data/preprocessed/may-3.csv', encoding='ISO-8859-1')

def count_if_exist(corpus, word):
    word_count = corpus.get(word,0) + 1
    corpus[word] = word_count

def convert_dict_to_csv(input_dict,name):
    pd.DataFrame(input_dict.items()).to_csv("./data/computed/{name}.csv".format(name=name),header=None,index=None)
general_word_count = {}
supporting_word_count = {} 
non_supporting_word_count = {}
word_support_probability = {}
for i in range(1,len(df)):
    cur_row = df.iloc[i]
    words = cur_row.text.split(' ')
    for word in words:
        count_if_exist(general_word_count,word)
        if(cur_row['Y/N'] == 'Y'):
            count_if_exist(supporting_word_count,word)
        else:
            count_if_exist(non_supporting_word_count,word)
print(general_word_count)
"""
convert_dict_to_csv(general_word_count,"general_word_count")
convert_dict_to_csv(supporting_word_count,"supporting_word_count")
convert_dict_to_csv(non_supporting_word_count,"non_supporting_word_count")
"""

"""
for word, word_count in general_word_count.items():
    if(word_count > 5):
        if(word in supporting_word_count):
            word_support_probability[word] = supporting_word_count[word] / word_count
        elif(word in non_supporting_word_count):
            word_support_probability[word] = 1 - non_supporting_word_count[word] / word_count
   """