# coding: utf-8

# Extracting the data
# ==
# We proceed to get the data from the arff file.

# In[276]:

# Let's load the data
import numpy as np
import pandas as pd
from pandas import DataFrame
from scipy.io.arff import loadarff
from sklearn.preprocessing import LabelEncoder

# printing options
np.set_printoptions(precision=4)
np.set_printoptions(suppress=True)

def arfftosk(filename):
    """Function to convert a .arff file to sklearn-compatible format

    Keyword arguments:
    filename -- the name of the file
    """

    data_raw, metadata_raw = loadarff(filename)
    df = pd.DataFrame(data_raw)
    # numpy notation for multiple columns, select everything but the last column
    X_df = df.iloc[:,:-1]
    # last column is the class
    y_df = df.iloc[:,-1]
    y_encoder = LabelEncoder()
    X = X_df.values
    y = y_encoder.fit_transform(y_df)


    return X,y,y_encoder


# In[277]:

X_train,y_train, y_encoder = arfftosk("CE802_Ass_2015_Data.arff")
y_encoder.classes_


# ### Array contents
# * Let's see what we have loaded
# * Notice how missing data are represented as "nan" values

# In[278]:

import math
# Let's check the missing values
X_dframe = DataFrame(X_train)
X_dframe.replace(to_replace=float('nan'), value=np.nan, inplace=True)
X_dframe.shape


# Imputation
# ===
# * What if we have missing values for some data column?
# * Not all classifiers/regressors work with missing values
# * Scikit-learn provides methods for *imputing* missing values into the data
#
# (credits: made by Spyros Samothrakis and Luca Citi (regressors))

# ### Remove missing values
# First we try to remove all the rows with missing values.

# In[279]:

# Let's select the columns to be used as regressors
regressee = 19
regressors = range(X_train.shape[1])
regressors.remove(regressee)


# In[280]:

'''
Let's select the instances that can be used to learn to predict
missing values from the given column.
'''
# np.isfinite creates an nparray with all the rows that don't have missing values
valid = np.isfinite(X_train[:,regressee])
X_train_no_missing = X_train[valid]
y_train_no_missing = y_train[valid]
X_train_no_missing.shape


# ### Mean Imputation
# * Using an Imputer with the defualt values, we can just replace
# * all occurrences of NaN (the python value) with the mean for
# * that column.

# In[281]:

# Imputation, replace every missing column with the mean
from sklearn.preprocessing import Imputer
imp = Imputer()
X_train_imp_mean = imp.fit_transform(X_train)

# Show the results in the notebook, notice that there are no NaNs
# left in the column 19 or the 20th feature.
mean_dframe = DataFrame(X_train_imp_mean)
mean_dframe


# ### Linear Regressor Imputation
# * We can use linear regression to impute the missing data

# In[282]:

# Let's train a linear model!
from sklearn.linear_model import LinearRegression as lr
imp = lr()
# ix returns an array of cross products of the 1D arrays sent to
# it as parameters. So, the cross product of booleans with all
# regressors, so: X_Train[False, 0], ..., X_Train[False, 18]
# This is done for each False in the valid array.
# So you end up with
imp.fit(X_train[np.ix_(valid,regressors)], X_train[valid,regressee])
# ... and use it to predict
imputed_linear = imp.predict(X_train[:,regressors])


# In[283]:

# Let's see how it performs on the data
get_ipython().magic(u'matplotlib inline')
import matplotlib.pyplot as plt
plt.figure(figsize=(10,8))
# picking the first 50 values to be able to see the result in the plot
plt.plot(X_train[200:251,regressee])
plt.plot(imputed_linear[200:251])
plt.show()


# In[284]:

# Let's replace the imputed values in place of the missing value
X_train_imp_linear = X_train.copy()
X_train_imp_linear[~valid,regressee] = imputed_linear[~valid]
# ... and plot the results
plt.figure(figsize=(8,6))
plt.plot(X_train_imp_linear[:,regressee])
plt.show()


# ### Random Forest Regressor Imputation

# In[285]:

# Let's train a linear model!
from sklearn.ensemble import RandomForestRegressor

imp = RandomForestRegressor(random_state=0, n_estimators=100)

imp.fit(X_train[np.ix_(valid,regressors)], X_train[valid,regressee])
# ... and use it to predict
imputed_random_forest = imp.predict(X_train[:,regressors])


# In[286]:

plt.figure(figsize=(10,8))
# picking the first 50 values to be able to see the result in the plot
plt.plot(X_train[200:251,regressee])
plt.plot(imputed_random_forest[200:251])
plt.show()


# In[287]:

# Let's replace the imputed values in place of the missing value
X_train_imp_forest = X_train.copy()
X_train_imp_forest[~valid,regressee] = imputed_random_forest[~valid]
# ... and plot the results
plt.figure(figsize=(8,6))
plt.plot(X_train_imp_forest[:,regressee])
plt.show()


# ### Plotting all the imputations to compare them

# In[288]:

plt.figure(figsize=(10,10))
plt.plot(X_train[150:251,regressee])
plt.plot(imputed_linear[150:251])
plt.plot(imputed_random_forest[150:251])
plt.legend(["non", "linear", "random forests"])
plt.show()


# ### Comparing imputation methods
# In order to compare imputation methods, crossvalidation can be used, and their kappa statistic can be observed. Will do this if I have the time.

# Training a model
# ===
# Now that Imputation is done, it is possible to train a model to classify the last variable. This has to be a classifier since the last variable is binary.
#
# We are going to use the following
#     1. Naive Bayes: Simple model to prevent overfitting.
#     2. Decision Trees: With post prunning to reduce overfitting.
#     3. Random Forest Classifiers: Make our decision tree model more general.
#
# But first, lets setup the global variables that state the k for k-fold cross validation and the max depth for both of the tree algorithms.

# In[289]:

cv_folds = 10
tree_max_depth = None


# ### Naive Bayes
# We will use Gaussian Naive Bayes to classify our data first.
#
# We will check how it performs on:
# * Data without missing values X_train_no_missing
# * Data with mean imputated missing values X_train_imp_mean
# * Data with imputated mising values with linear regression X_train_imp_linear
# * Data with imputated missing values with a Random Forest Regressor X_train_imp_forest

# In[293]:

from sklearn.naive_bayes import GaussianNB
from sklearn import cross_validation as cv
from sklearn import metrics
clf = GaussianNB()

# Remove missing values
bayes_nomissing_pred = cv.cross_val_predict(clf,
                        X_train_no_missing, y_train_no_missing,
                        cv=cv_folds)

bayes_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
                                             bayes_nomissing_pred)

bayes_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
                                              bayes_nomissing_pred)

# Imputation with mean
bayes_mean_pred = cv.cross_val_predict(clf,
                        X_train_imp_mean, y_train,
                        cv=cv_folds)

bayes_mean_acc = metrics.accuracy_score(y_train,
                                             bayes_mean_pred)

bayes_mean_kap = metrics.cohen_kappa_score(y_train,
                                              bayes_mean_pred)

# Imputation with linear regression
bayes_linear_pred = cv.cross_val_predict(clf,
                        X_train_imp_linear, y_train,
                        cv=cv_folds)

bayes_linear_acc = metrics.accuracy_score(y_train,
                                             bayes_linear_pred)

bayes_linear_kap = metrics.cohen_kappa_score(y_train,
                                              bayes_linear_pred)

# Imputation with random forest regressor
bayes_forest_pred = cv.cross_val_predict(clf,
                        X_train_imp_forest, y_train,
                        cv=cv_folds)

bayes_forest_acc = metrics.accuracy_score(y_train,
                                             bayes_forest_pred)

bayes_forest_kap = metrics.cohen_kappa_score(y_train,
                                              bayes_forest_pred)

# No cross validation
clf.fit(X_train_imp_forest, y_train)
bayes_pred = clf.predict(X_train_imp_forest)

bayes_acc = metrics.accuracy_score(y_train, bayes_pred)

bayes_kap = metrics.cohen_kappa_score(y_train, bayes_pred)
print "\nNo cross validation + Bayes + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (bayes_acc,
                                 bayes_kap)
print metrics.confusion_matrix(y_train, bayes_pred)

print "\nBayes + Remove missing values"
print "Acc: %.4f Kappa: %.4f" % (bayes_nomissing_acc,
                                 bayes_nomissing_kap)
print metrics.confusion_matrix(y_train_no_missing, bayes_nomissing_pred)

print "\nBayes + Impute with Mean"
print "Acc: %.4f Kappa: %.4f" % (bayes_mean_acc,
                                 bayes_mean_kap)
print metrics.confusion_matrix(y_train, bayes_mean_pred)

print "\nBayes + Impute with Linear Regressor"
print "Acc: %.4f Kappa: %.4f" % (bayes_linear_acc,
                                 bayes_linear_kap)
print metrics.confusion_matrix(y_train, bayes_linear_pred)

print "\nBayes + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (bayes_forest_acc,
                                 bayes_forest_kap)
print metrics.confusion_matrix(y_train, bayes_forest_pred)


# ### Decision Tree

# In[294]:

from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier(random_state=0, max_depth=tree_max_depth)

# Remove missing values
tree_nomissing_pred = cv.cross_val_predict(clf,
                        X_train_no_missing, y_train_no_missing,
                        cv=cv_folds)

tree_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
                                             tree_nomissing_pred)

tree_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
                                              tree_nomissing_pred)

# Imputation with mean
tree_mean_pred = cv.cross_val_predict(clf,
                        X_train_imp_mean, y_train,
                        cv=cv_folds)

tree_mean_acc = metrics.accuracy_score(y_train,
                                             tree_mean_pred)

tree_mean_kap = metrics.cohen_kappa_score(y_train,
                                              tree_mean_pred)

# Imputation with linear regression
tree_linear_pred = cv.cross_val_predict(clf,
                        X_train_imp_linear, y_train,
                        cv=cv_folds)

tree_linear_acc = metrics.accuracy_score(y_train,
                                             tree_linear_pred)

tree_linear_kap = metrics.cohen_kappa_score(y_train,
                                              tree_linear_pred)

# Imputation with random forest regressor
tree_forest_pred = cv.cross_val_predict(clf,
                        X_train_imp_forest, y_train,
                        cv=cv_folds)

tree_forest_acc = metrics.accuracy_score(y_train,
                                             tree_forest_pred)

tree_forest_kap = metrics.cohen_kappa_score(y_train,
                                              tree_forest_pred)

# No cross validation
clf.fit(X_train_imp_forest, y_train)
tree_pred = clf.predict(X_train_imp_forest)

tree_acc = metrics.accuracy_score(y_train, tree_pred)

tree_kap = metrics.cohen_kappa_score(y_train, tree_pred)
print "\nNo cross validation + Decision Tree + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (tree_acc,
                                 tree_kap)
print metrics.confusion_matrix(y_train, tree_pred)

print "\nDecision Tree + Remove missing values"
print "Acc: %.4f Kappa: %.4f" % (tree_nomissing_acc,
                                 tree_nomissing_kap)
print metrics.confusion_matrix(y_train_no_missing, tree_nomissing_pred)

print "\nDecision Tree + Impute with Mean"
print "Acc: %.4f Kappa: %.4f" % (tree_mean_acc,
                                 tree_mean_kap)
print metrics.confusion_matrix(y_train, tree_mean_pred)

print "\nDecision Tree + Impute with Linear Regressor"
print "Acc: %.4f Kappa: %.4f" % (tree_linear_acc,
                                 tree_linear_kap)
print metrics.confusion_matrix(y_train, tree_linear_pred)

print "\nDecision Tree + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (tree_forest_acc,
                                 tree_forest_kap)
print metrics.confusion_matrix(y_train, tree_forest_pred)


# ### Random Forest Classifier

# In[292]:

from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(n_estimators=100,
                             random_state=0,
                             max_depth=tree_max_depth)

# Remove missing values
forest_nomissing_pred = cv.cross_val_predict(clf,
                        X_train_no_missing, y_train_no_missing,
                        cv=cv_folds)

forest_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
                                             forest_nomissing_pred)

forest_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
                                              forest_nomissing_pred)

# Imputation with mean
forest_mean_pred = cv.cross_val_predict(clf,
                        X_train_imp_mean, y_train,
                        cv=cv_folds)

forest_mean_acc = metrics.accuracy_score(y_train,
                                             forest_mean_pred)

forest_mean_kap = metrics.cohen_kappa_score(y_train,
                                              forest_mean_pred)

# Imputation with linear regression
forest_linear_pred = cv.cross_val_predict(clf,
                        X_train_imp_linear, y_train,
                        cv=cv_folds)

forest_linear_acc = metrics.accuracy_score(y_train,
                                             forest_linear_pred)

forest_linear_kap = metrics.cohen_kappa_score(y_train,
                                              forest_linear_pred)

# Imputation with random forest regressor
forest_forest_pred = cv.cross_val_predict(clf,
                        X_train_imp_forest, y_train,
                        cv=cv_folds)

forest_forest_acc = metrics.accuracy_score(y_train,
                                             forest_forest_pred)

forest_forest_kap = metrics.cohen_kappa_score(y_train,
                                              forest_forest_pred)

# No cross validation
clf.fit(X_train_imp_forest, y_train)
forest_pred = clf.predict(X_train_imp_forest)

forest_acc = metrics.accuracy_score(y_train, forest_pred)

forest_kap = metrics.cohen_kappa_score(y_train, forest_pred)
print "\nNo cross validation + Random Forest Classifier + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (forest_acc,
                                 forest_kap)
print metrics.confusion_matrix(y_train, forest_pred)

print "\nRandom Forest Classifier + Remove missing values"
print "Acc: %.4f Kappa: %.4f" % (forest_nomissing_acc,
                                 forest_nomissing_kap)
print metrics.confusion_matrix(y_train_no_missing,
                               forest_nomissing_pred)

print "\nRandom Forest Classifier + Impute with Mean"
print "Acc: %.4f Kappa: %.4f" % (forest_mean_acc,
                                 forest_mean_kap)
print metrics.confusion_matrix(y_train, forest_mean_pred)

print "\nRandom Forest Classifier + Impute with Linear Regressor"
print "Acc: %.4f Kappa: %.4f" % (forest_linear_acc,
                                 forest_linear_kap)
print metrics.confusion_matrix(y_train, forest_linear_pred)

print "\nRandom Forest Classifier + Impute with Random Forest Regressor"
print "Acc: %.4f Kappa: %.4f" % (forest_forest_acc,
                                 forest_forest_kap)
print metrics.confusion_matrix(y_train, forest_forest_pred)