preprocessing.py

import pandas as pd
import numpy as np


def get_data(data_file):
    data = pd.read_csv(data_file)
    data = data.drop(columns=['ID'])

    return data


def get_data_with_id(data_file):
    data = pd.read_csv(data_file)

    return data


# imputation
def get_average_column_value(data, col):
    total = 0
    count = 0

    for i in range(data.shape[0]):
        if data[i][col] != '?':
            total += int(data[i][col])
            count += 1

    return int(round(total/count, 0))


def process_missing_values(data, remove=True):
    vals = data.values
    new_vals = np.zeros(vals.shape)

    diff = 0

    for i in range(vals.shape[0]):
        for j in range(len(vals[i])):
            if vals[i][j] == '?':
                if remove:
                    new_vals = np.delete(new_vals, i-diff, 0)
                    diff += 1
                else:
                    new_vals[i-diff][j] = get_average_column_value(vals, j)
            elif isinstance(vals[i][j], str):
                new_vals[i-diff][j] = int(vals[i][j])
            else:
                new_vals[i-diff][j] = vals[i][j]

    return new_vals


def preprocess():
    breastcancer_data = get_data('breast-cancer-wisconsin.data')

    removed_data = process_missing_values(breastcancer_data, remove=True)
    removed = pd.DataFrame(removed_data)
    removed.columns = breastcancer_data.columns

    average_data = process_missing_values(breastcancer_data, remove=False)
    average = pd.DataFrame(average_data)
    average.columns = breastcancer_data.columns

    return [breastcancer_data, removed, average]