cluster.py

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from imblearn.over_sampling import RandomOverSampler
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import MinMaxScaler
from pandas import read_csv,DataFrame
from collections import Counter
from numpy import fromiter
import numpy
import os
import csv
import config

cols = [
    'Y/N',
    'favorite_count',
    'retweet_count',
    'statuses_count',
    'friends_count',
    'hashtag_count',
    'capitals',
    'exclamation_marks',
    'sentiment',
    'readability',
    'topic',
]
def write_extracted_file(file_name,features):
    DataFrame(features).to_csv(
        'data/extracted/may-3/{file_name}'.format(file_name=file_name), 
        header=cols,
        index=None)

def cluster(tf_idf,n_clusters=3):
    print(n_clusters)
    kmeans = KMeans(init="k-means++",
       n_clusters=n_clusters,
        n_init=30,
        max_iter=500,
       random_state=42).fit(tf_idf)
    print(Counter(kmeans.labels_))
    return kmeans.labels_

def extract_features():  
    df = read_csv('data/preprocessed/april-21.csv')
    X = df.drop(columns=['text'],axis=1).values

    for i in config.grams:
        tfidf_vectorizer = TfidfVectorizer(ngram_range=(i,i))
        tf_idf = tfidf_vectorizer.fit_transform(df['text'].values.astype('U'))
        for num_cluster in config.clusters:
            kmeans = KMeans(init="k-means++",
            n_clusters=num_cluster,
                n_init=30,
                max_iter=500,
            random_state=42).fit(tf_idf)
       
            my_cluster = kmeans.labels_
            Y = my_cluster
            #oversample = RandomOverSampler(sampling_strategy='all')
            #new_X, new_Y = oversample.fit_resample(X,Y)

            np_Y = numpy.array([Y])
            unscaled_features = numpy.concatenate((X,np_Y.T),axis=1)
            scaler = MinMaxScaler(feature_range=(0, 1))
            rescaled_features = scaler.fit_transform(unscaled_features)
            
            df['cluster'] = my_cluster
            df[['text','cluster','Y/N']].to_csv('data/computed/cluster/april-21/{i}-gram-{num_cluster}-clusters.csv'.format(i=i,num_cluster=num_cluster),index=None)
            #write_extracted_file('{i}-gram-{num_cluster}-clusters.csv'.format(i=i,num_cluster=num_cluster),rescaled_features)
            
if __name__  == "__main__":
    extract_features()