whitespace_location_cleared.py

import string
import re
import matplotlib.pyplot as plt
import numpy as np
from collections import Counter
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import re
#read file

df = pd.read_csv(r"clean_file.csv", encoding ="latin-1")
cities_df = pd.read_csv(r"uscitiesv1.5.csv", encoding ="latin-1")

nan_value = float("NaN")

cities_df["full_county"] = (cities_df["county_name"] + ", " + cities_df["state_id"]).astype(str).str.lower()
cities_df["full_city"] = (cities_df["city"] + ", " + cities_df["state_id"]).astype(str).str.lower()
full_city = cities_df["full_city"].tolist()
full_county = cities_df["full_county"].tolist()
df["location"] = df["location"].astype(str).str.lower()

def filter_location(word):
    
    if (word in full_county):
        return word
    elif (word in full_city):
        return word
    else:
        return nan_value


df['location'] = df['location'].apply(lambda x: filter_location(x))

df = df.dropna(subset=['location'])
df = df.reset_index(drop=True)
df['tweet'] = df['tweet'].apply(lambda x: " ".join(str(x).split()))

df.to_csv("clean_file_test.csv")
df.head()