Delta TFIDF, part 1

BagOfWords.py

@@ -2,7 +2,7 @@ from __future__ import division
 import string
 import numpy
 import nltk
-from TFIDF import tfidf
+from TFIDF import tfidf, delta_tfidf
 
 # "Adapting a technique of Das and Chen (2001), we added the tag NOT to every word between a negation word ('not',
 # 'isn't', 'didn't', etc.) and the first punctuation mark following the negation word."
@@ -81,7 +81,29 @@ def make_tfidf(document, documents):
   for key in bag.keys():
     bag[key] /= factor
   return bag
-
+
+# As per Martineau and Finn (2009), create a bag of words using delta TFIDF as the feature value.
+# Todo: Bigrams?
+def make_delta_tfidf(document, positive_set, negative_set, ref_bag):
+  bag = {}
+  factor = 0
+  for term in set(document):
+    weight = delta_tfidf(term, document, positive_set, negative_set)
+    if (weight != 0):
+      bag[term] = weight
+      factor += weight**2
+  factor **= 0.5
+  for key in bag.keys():
+    bag[key] /= factor
+  # Add word counts to the reference bag
+  for term in document:
+    if ref_bag != None:
+      if ref_bag.has_key(term):
+        ref_bag[term] += 1
+      else:
+        ref_bag[term] = 1
+  return bag
+
 def to_vector(bag, wordlist):
   vec = []
   for word in wordlist: