# coding: utf-8
# Extracting the data
# ==
# We proceed to get the data from the arff file.
# In[276]:
# Let's load the data
import numpy as np
import pandas as pd
from pandas import DataFrame
from scipy.io.arff import loadarff
from sklearn.preprocessing import LabelEncoder
# printing options
np.set_printoptions(precision=4)
np.set_printoptions(suppress=True)
def arfftosk(filename):
"""Function to convert a .arff file to sklearn-compatible format
Keyword arguments:
filename -- the name of the file
"""
data_raw, metadata_raw = loadarff(filename)
df = pd.DataFrame(data_raw)
# numpy notation for multiple columns, select everything but the last column
X_df = df.iloc[:,:-1]
# last column is the class
y_df = df.iloc[:,-1]
y_encoder = LabelEncoder()
X = X_df.values
y = y_encoder.fit_transform(y_df)
return X,y,y_encoder
# In[277]:
X_train,y_train, y_encoder = arfftosk("CE802_Ass_2015_Data.arff")
y_encoder.classes_
# ### Array contents
# * Let's see what we have loaded
# * Notice how missing data are represented as "nan" values
# In[278]:
import math
# Let's check the missing values
X_dframe = DataFrame(X_train)
X_dframe.replace(to_replace=float('nan'), value=np.nan, inplace=True)
X_dframe.shape
# Imputation
# ===
# * What if we have missing values for some data column?
# * Not all classifiers/regressors work with missing values
# * Scikit-learn provides methods for *imputing* missing values into the data
#
# (credits: made by Spyros Samothrakis and Luca Citi (regressors))
# ### Remove missing values
# First we try to remove all the rows with missing values.
# In[279]:
# Let's select the columns to be used as regressors
regressee = 19
regressors = range(X_train.shape[1])
regressors.remove(regressee)
# In[280]:
'''
Let's select the instances that can be used to learn to predict
missing values from the given column.
'''
# np.isfinite creates an nparray with all the rows that don't have missing values
valid = np.isfinite(X_train[:,regressee])
X_train_no_missing = X_train[valid]
y_train_no_missing = y_train[valid]
X_train_no_missing.shape
# ### Mean Imputation
# * Using an Imputer with the defualt values, we can just replace
# * all occurrences of NaN (the python value) with the mean for
# * that column.
# In[281]:
# Imputation, replace every missing column with the mean
from sklearn.preprocessing import Imputer
imp = Imputer()
X_train_imp_mean = imp.fit_transform(X_train)
# Show the results in the notebook, notice that there are no NaNs
# left in the column 19 or the 20th feature.
mean_dframe = DataFrame(X_train_imp_mean)
mean_dframe
# ### Linear Regressor Imputation
# * We can use linear regression to impute the missing data
# In[282]:
# Let's train a linear model!
from sklearn.linear_model import LinearRegression as lr
imp = lr()
# ix returns an array of cross products of the 1D arrays sent to
# it as parameters. So, the cross product of booleans with all
# regressors, so: X_Train[False, 0], ..., X_Train[False, 18]
# This is done for each False in the valid array.
# So you end up with
imp.fit(X_train[np.ix_(valid,regressors)], X_train[valid,regressee])
# ... and use it to predict
imputed_linear = imp.predict(X_train[:,regressors])
# In[283]:
# Let's see how it performs on the data
get_ipython().magic(u'matplotlib inline')
import matplotlib.pyplot as plt
plt.figure(figsize=(10,8))
# picking the first 50 values to be able to see the result in the plot
plt.plot(X_train[200:251,regressee])
plt.plot(imputed_linear[200:251])
plt.show()
# In[284]:
# Let's replace the imputed values in place of the missing value
X_train_imp_linear = X_train.copy()
X_train_imp_linear[~valid,regressee] = imputed_linear[~valid]
# ... and plot the results
plt.figure(figsize=(8,6))
plt.plot(X_train_imp_linear[:,regressee])
plt.show()
# ### Random Forest Regressor Imputation
# In[285]:
# Let's train a linear model!
from sklearn.ensemble import RandomForestRegressor
imp = RandomForestRegressor(random_state=0, n_estimators=100)
imp.fit(X_train[np.ix_(valid,regressors)], X_train[valid,regressee])
# ... and use it to predict
imputed_random_forest = imp.predict(X_train[:,regressors])
# In[286]:
plt.figure(figsize=(10,8))
# picking the first 50 values to be able to see the result in the plot
plt.plot(X_train[200:251,regressee])
plt.plot(imputed_random_forest[200:251])
plt.show()
# In[287]:
# Let's replace the imputed values in place of the missing value
X_train_imp_forest = X_train.copy()
X_train_imp_forest[~valid,regressee] = imputed_random_forest[~valid]
# ... and plot the results
plt.figure(figsize=(8,6))
plt.plot(X_train_imp_forest[:,regressee])
plt.show()
# ### Plotting all the imputations to compare them
# In[288]:
plt.figure(figsize=(10,10))
plt.plot(X_train[150:251,regressee])
plt.plot(imputed_linear[150:251])
plt.plot(imputed_random_forest[150:251])
plt.legend(["non", "linear", "random forests"])
plt.show()
# ### Comparing imputation methods
# In order to compare imputation methods, crossvalidation can be used, and their kappa statistic can be observed. Will do this if I have the time.
# Training a model
# ===
# Now that Imputation is done, it is possible to train a model to classify the last variable. This has to be a classifier since the last variable is binary.
#
# We are going to use the following
# 1. Naive Bayes: Simple model to prevent overfitting.
# 2. Decision Trees: With post prunning to reduce overfitting.
# 3. Random Forest Classifiers: Make our decision tree model more general.
#
# But first, lets setup the global variables that state the k for k-fold cross validation and the max depth for both of the tree algorithms.
# In[289]:
cv_folds = 10
tree_max_depth = None
# ### Naive Bayes
# We will use Gaussian Naive Bayes to classify our data first.
#
# We will check how it performs on:
# * Data without missing values X_train_no_missing
# * Data with mean imputated missing values X_train_imp_mean
# * Data with imputated mising values with linear regression X_train_imp_linear
# * Data with imputated missing values with a Random Forest Regressor X_train_imp_forest
# In[293]:
from sklearn.naive_bayes import GaussianNB
from sklearn import cross_validation as cv
from sklearn import metrics
clf = GaussianNB()
# Remove missing values
bayes_nomissing_pred = cv.cross_val_predict(clf,
X_train_no_missing, y_train_no_missing,
cv=cv_folds)
bayes_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
bayes_nomissing_pred)
bayes_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
bayes_nomissing_pred)
# Imputation with mean
bayes_mean_pred = cv.cross_val_predict(clf,
X_train_imp_mean, y_train,
cv=cv_folds)
bayes_mean_acc = metrics.accuracy_score(y_train,
bayes_mean_pred)
bayes_mean_kap = metrics.cohen_kappa_score(y_train,
bayes_mean_pred)
# Imputation with linear regression
bayes_linear_pred = cv.cross_val_predict(clf,
X_train_imp_linear, y_train,
cv=cv_folds)
bayes_linear_acc = metrics.accuracy_score(y_train,
bayes_linear_pred)
bayes_linear_kap = metrics.cohen_kappa_score(y_train,
bayes_linear_pred)
# Imputation with random forest regressor
bayes_forest_pred = cv.cross_val_predict(clf,
X_train_imp_forest, y_train,
cv=cv_folds)
bayes_forest_acc = metrics.accuracy_score(y_train,
bayes_forest_pred)
bayes_forest_kap = metrics.cohen_kappa_score(y_train,
bayes_forest_pred)
# No cross validation
clf.fit(X_train_imp_forest, y_train)
bayes_pred = clf.predict(X_train_imp_forest)
bayes_acc = metrics.accuracy_score(y_train, bayes_pred)
bayes_kap = metrics.cohen_kappa_score(y_train, bayes_pred)
print "\nNo cross validation + Bayes + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (bayes_acc,
bayes_kap)
print metrics.confusion_matrix(y_train, bayes_pred)
print "\nBayes + Remove missing values"
print "Acc: %.4f Kappa: %.4f" % (bayes_nomissing_acc,
bayes_nomissing_kap)
print metrics.confusion_matrix(y_train_no_missing, bayes_nomissing_pred)
print "\nBayes + Impute with Mean"
print "Acc: %.4f Kappa: %.4f" % (bayes_mean_acc,
bayes_mean_kap)
print metrics.confusion_matrix(y_train, bayes_mean_pred)
print "\nBayes + Impute with Linear Regressor"
print "Acc: %.4f Kappa: %.4f" % (bayes_linear_acc,
bayes_linear_kap)
print metrics.confusion_matrix(y_train, bayes_linear_pred)
print "\nBayes + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (bayes_forest_acc,
bayes_forest_kap)
print metrics.confusion_matrix(y_train, bayes_forest_pred)
# ### Decision Tree
# In[294]:
from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier(random_state=0, max_depth=tree_max_depth)
# Remove missing values
tree_nomissing_pred = cv.cross_val_predict(clf,
X_train_no_missing, y_train_no_missing,
cv=cv_folds)
tree_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
tree_nomissing_pred)
tree_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
tree_nomissing_pred)
# Imputation with mean
tree_mean_pred = cv.cross_val_predict(clf,
X_train_imp_mean, y_train,
cv=cv_folds)
tree_mean_acc = metrics.accuracy_score(y_train,
tree_mean_pred)
tree_mean_kap = metrics.cohen_kappa_score(y_train,
tree_mean_pred)
# Imputation with linear regression
tree_linear_pred = cv.cross_val_predict(clf,
X_train_imp_linear, y_train,
cv=cv_folds)
tree_linear_acc = metrics.accuracy_score(y_train,
tree_linear_pred)
tree_linear_kap = metrics.cohen_kappa_score(y_train,
tree_linear_pred)
# Imputation with random forest regressor
tree_forest_pred = cv.cross_val_predict(clf,
X_train_imp_forest, y_train,
cv=cv_folds)
tree_forest_acc = metrics.accuracy_score(y_train,
tree_forest_pred)
tree_forest_kap = metrics.cohen_kappa_score(y_train,
tree_forest_pred)
# No cross validation
clf.fit(X_train_imp_forest, y_train)
tree_pred = clf.predict(X_train_imp_forest)
tree_acc = metrics.accuracy_score(y_train, tree_pred)
tree_kap = metrics.cohen_kappa_score(y_train, tree_pred)
print "\nNo cross validation + Decision Tree + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (tree_acc,
tree_kap)
print metrics.confusion_matrix(y_train, tree_pred)
print "\nDecision Tree + Remove missing values"
print "Acc: %.4f Kappa: %.4f" % (tree_nomissing_acc,
tree_nomissing_kap)
print metrics.confusion_matrix(y_train_no_missing, tree_nomissing_pred)
print "\nDecision Tree + Impute with Mean"
print "Acc: %.4f Kappa: %.4f" % (tree_mean_acc,
tree_mean_kap)
print metrics.confusion_matrix(y_train, tree_mean_pred)
print "\nDecision Tree + Impute with Linear Regressor"
print "Acc: %.4f Kappa: %.4f" % (tree_linear_acc,
tree_linear_kap)
print metrics.confusion_matrix(y_train, tree_linear_pred)
print "\nDecision Tree + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (tree_forest_acc,
tree_forest_kap)
print metrics.confusion_matrix(y_train, tree_forest_pred)
# ### Random Forest Classifier
# In[292]:
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(n_estimators=100,
random_state=0,
max_depth=tree_max_depth)
# Remove missing values
forest_nomissing_pred = cv.cross_val_predict(clf,
X_train_no_missing, y_train_no_missing,
cv=cv_folds)
forest_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
forest_nomissing_pred)
forest_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
forest_nomissing_pred)
# Imputation with mean
forest_mean_pred = cv.cross_val_predict(clf,
X_train_imp_mean, y_train,
cv=cv_folds)
forest_mean_acc = metrics.accuracy_score(y_train,
forest_mean_pred)
forest_mean_kap = metrics.cohen_kappa_score(y_train,
forest_mean_pred)
# Imputation with linear regression
forest_linear_pred = cv.cross_val_predict(clf,
X_train_imp_linear, y_train,
cv=cv_folds)
forest_linear_acc = metrics.accuracy_score(y_train,
forest_linear_pred)
forest_linear_kap = metrics.cohen_kappa_score(y_train,
forest_linear_pred)
# Imputation with random forest regressor
forest_forest_pred = cv.cross_val_predict(clf,
X_train_imp_forest, y_train,
cv=cv_folds)
forest_forest_acc = metrics.accuracy_score(y_train,
forest_forest_pred)
forest_forest_kap = metrics.cohen_kappa_score(y_train,
forest_forest_pred)
# No cross validation
clf.fit(X_train_imp_forest, y_train)
forest_pred = clf.predict(X_train_imp_forest)
forest_acc = metrics.accuracy_score(y_train, forest_pred)
forest_kap = metrics.cohen_kappa_score(y_train, forest_pred)
print "\nNo cross validation + Random Forest Classifier + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (forest_acc,
forest_kap)
print metrics.confusion_matrix(y_train, forest_pred)
print "\nRandom Forest Classifier + Remove missing values"
print "Acc: %.4f Kappa: %.4f" % (forest_nomissing_acc,
forest_nomissing_kap)
print metrics.confusion_matrix(y_train_no_missing,
forest_nomissing_pred)
print "\nRandom Forest Classifier + Impute with Mean"
print "Acc: %.4f Kappa: %.4f" % (forest_mean_acc,
forest_mean_kap)
print metrics.confusion_matrix(y_train, forest_mean_pred)
print "\nRandom Forest Classifier + Impute with Linear Regressor"
print "Acc: %.4f Kappa: %.4f" % (forest_linear_acc,
forest_linear_kap)
print metrics.confusion_matrix(y_train, forest_linear_pred)
print "\nRandom Forest Classifier + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (forest_forest_acc,
forest_forest_kap)
print metrics.confusion_matrix(y_train, forest_forest_pred)
Comments