Titanic Data Analysis_MachineLearning_DeepLearning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from keras.constraints import maxnorm\n",
    "from keras.optimizers import SGD\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from keras import optimizers\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_raw = pd.read_csv('train.csv')\n",
    "testing_raw = pd.read_csv('test.csv')\n",
    "combine = [training_raw, testing_raw]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>714.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.383838</td>\n",
       "      <td>2.308642</td>\n",
       "      <td>29.699118</td>\n",
       "      <td>0.523008</td>\n",
       "      <td>0.381594</td>\n",
       "      <td>32.204208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>257.353842</td>\n",
       "      <td>0.486592</td>\n",
       "      <td>0.836071</td>\n",
       "      <td>14.526497</td>\n",
       "      <td>1.102743</td>\n",
       "      <td>0.806057</td>\n",
       "      <td>49.693429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>223.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.125000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.910400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.454200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>668.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>31.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>512.329200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       PassengerId    Survived      Pclass         Age       SibSp  \\\n",
       "count   891.000000  891.000000  891.000000  714.000000  891.000000   \n",
       "mean    446.000000    0.383838    2.308642   29.699118    0.523008   \n",
       "std     257.353842    0.486592    0.836071   14.526497    1.102743   \n",
       "min       1.000000    0.000000    1.000000    0.420000    0.000000   \n",
       "25%     223.500000    0.000000    2.000000   20.125000    0.000000   \n",
       "50%     446.000000    0.000000    3.000000   28.000000    0.000000   \n",
       "75%     668.500000    1.000000    3.000000   38.000000    1.000000   \n",
       "max     891.000000    1.000000    3.000000   80.000000    8.000000   \n",
       "\n",
       "            Parch        Fare  \n",
       "count  891.000000  891.000000  \n",
       "mean     0.381594   32.204208  \n",
       "std      0.806057   49.693429  \n",
       "min      0.000000    0.000000  \n",
       "25%      0.000000    7.910400  \n",
       "50%      0.000000   14.454200  \n",
       "75%      0.000000   31.000000  \n",
       "max      6.000000  512.329200  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_raw.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>204</td>\n",
       "      <td>889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>891</td>\n",
       "      <td>2</td>\n",
       "      <td>681</td>\n",
       "      <td>147</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>Frost, Mr. Anthony Wood \"Archie\"</td>\n",
       "      <td>male</td>\n",
       "      <td>1601</td>\n",
       "      <td>C23 C25 C27</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>577</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>644</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Name   Sex Ticket        Cabin Embarked\n",
       "count                                891   891    891          204      889\n",
       "unique                               891     2    681          147        3\n",
       "top     Frost, Mr. Anthony Wood \"Archie\"  male   1601  C23 C25 C27        S\n",
       "freq                                   1   577      7            4      644"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_raw.describe(include = ['O'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PassengerId      0\n",
       "Survived         0\n",
       "Pclass           0\n",
       "Name             0\n",
       "Sex              0\n",
       "Age            177\n",
       "SibSp            0\n",
       "Parch            0\n",
       "Ticket           0\n",
       "Fare             0\n",
       "Cabin          687\n",
       "Embarked         2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#CHecking missing values\n",
    "training_raw.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PassengerId      0\n",
       "Pclass           0\n",
       "Name             0\n",
       "Sex              0\n",
       "Age             86\n",
       "SibSp            0\n",
       "Parch            0\n",
       "Ticket           0\n",
       "Fare             1\n",
       "Cabin          327\n",
       "Embarked         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_raw.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0.],\n",
       "       [0., 0., 0.]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#filling missing values with respect to gender and Pclass\n",
    "age_by_X = np.zeros(shape = [2,3])\n",
    "\n",
    "age_by_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "gender_mapping = {'male':0, 'female':1}\n",
    "for dataset in combine:\n",
    "    dataset['Sex'] = dataset['Sex'].map(gender_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name  Sex   Age  SibSp  Parch  \\\n",
       "0                            Braund, Mr. Owen Harris    0  22.0      1      0   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...    1  38.0      1      0   \n",
       "2                             Heikkinen, Miss. Laina    1  26.0      0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)    1  35.0      1      0   \n",
       "4                           Allen, Mr. William Henry    0  35.0      0      0   \n",
       "\n",
       "             Ticket     Fare Cabin Embarked  \n",
       "0         A/5 21171   7.2500   NaN        S  \n",
       "1          PC 17599  71.2833   C85        C  \n",
       "2  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3            113803  53.1000  C123        S  \n",
       "4            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_for_age = pd.concat([training_raw[['Sex', 'Age', 'Pclass']], testing_raw[['Sex', 'Age', 'Pclass']]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1309, 3)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_for_age.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for age imputataion\n",
    "for sex in range(2):\n",
    "    for pclass in range(3):\n",
    "        age_by_X[sex, pclass] = dataset_for_age.loc[(dataset_for_age['Sex'] == sex) & (dataset_for_age['Pclass'] == (pclass + 1)), 'Age' ].mean().round()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[41., 31., 26.],\n",
       "       [37., 27., 22.]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_by_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in combine:\n",
    "    for sex in range(2):\n",
    "        for pclass in range(3):\n",
    "            dataset.loc[(dataset['Age'].isnull()) & (dataset['Sex'] == sex) & (dataset['Pclass'] == (pclass + 1)), 'Age'] = age_by_X[sex, pclass]\n",
    "    dataset.loc[dataset['Embarked'].isnull(), 'Embarked'] = dataset['Embarked'].mode()[0]        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PassengerId      0\n",
       "Survived         0\n",
       "Pclass           0\n",
       "Name             0\n",
       "Sex              0\n",
       "Age              0\n",
       "SibSp            0\n",
       "Parch            0\n",
       "Ticket           0\n",
       "Fare             0\n",
       "Cabin          687\n",
       "Embarked         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_raw.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PassengerId      0\n",
       "Pclass           0\n",
       "Name             0\n",
       "Sex              0\n",
       "Age              0\n",
       "SibSp            0\n",
       "Parch            0\n",
       "Ticket           0\n",
       "Fare             1\n",
       "Cabin          327\n",
       "Embarked         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_raw.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>1044</td>\n",
       "      <td>3</td>\n",
       "      <td>Storey, Mr. Thomas</td>\n",
       "      <td>0</td>\n",
       "      <td>60.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3701</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     PassengerId  Pclass                Name  Sex   Age  SibSp  Parch Ticket  \\\n",
       "152         1044       3  Storey, Mr. Thomas    0  60.5      0      0   3701   \n",
       "\n",
       "     Fare Cabin Embarked  \n",
       "152   NaN   NaN        S  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#imputing Fare value in testing dataset\n",
    "testing_raw.loc[testing_raw['Fare'].isnull(), ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#now filling missing value base on pclass and gender\n",
    "testing_raw.loc[testing_raw['Fare'].isnull(), 'Fare'] = testing_raw.loc[(testing_raw['Sex'] == 0) & (testing_raw['Pclass'] == 3), 'Fare'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PassengerId      0\n",
       "Pclass           0\n",
       "Name             0\n",
       "Sex              0\n",
       "Age              0\n",
       "SibSp            0\n",
       "Parch            0\n",
       "Ticket           0\n",
       "Fare             0\n",
       "Cabin          327\n",
       "Embarked         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_raw.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#droping cabin because it has lots of missing values and dropping ticket because it has lots of duplicate values\n",
    "for dataset in combine:\n",
    "    dataset.drop(['Cabin', 'Ticket'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['PassengerId', 'Survived', 'Pclass', 'Name', 'Sex', 'Age', 'SibSp',\n",
       "       'Parch', 'Fare', 'Embarked'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_raw.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Pclass  Survived\n",
      "0       1  0.629630\n",
      "1       2  0.472826\n",
      "2       3  0.242363\n",
      "--------------------\n",
      "   Sex  Survived\n",
      "1    1  0.742038\n",
      "0    0  0.188908\n",
      "--------------------\n",
      "   SibSp  Survived\n",
      "1      1  0.535885\n",
      "2      2  0.464286\n",
      "0      0  0.345395\n",
      "3      3  0.250000\n",
      "4      4  0.166667\n",
      "5      5  0.000000\n",
      "6      8  0.000000\n",
      "--------------------\n",
      "   Parch  Survived\n",
      "3      3  0.600000\n",
      "1      1  0.550847\n",
      "2      2  0.500000\n",
      "0      0  0.343658\n",
      "5      5  0.200000\n",
      "4      4  0.000000\n",
      "6      6  0.000000\n",
      "--------------------\n",
      "  Embarked  Survived\n",
      "0        C  0.553571\n",
      "1        Q  0.389610\n",
      "2        S  0.339009\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "# here Pclass, Sex, Sibsp, Parch, Embarked are categorical so checking whether do they have any effect on Survived.\n",
    "cols_categorical = ['Pclass', 'Sex', 'SibSp', 'Parch', 'Embarked']\n",
    "for cols in cols_categorical:\n",
    "    print(training_raw[['Survived', cols]].groupby(cols, as_index = False)['Survived'].mean().sort_values(by = 'Survived',\n",
    "                                                                                                  ascending = False))\n",
    "    print('-'*20)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''from above it seems that all of the deatures had some realation with Survived except for Sibsp and Parch. Since Sibsp and \n",
    "Parch repsresents family so lets make a new feature called family and include all of them in it.'''\n",
    "for dataset in combine:\n",
    "    dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1 #1 because to add the person itself\n",
    "    dataset.drop(['SibSp', 'Parch'], axis = 1, inplace = True)\n",
    "    #also creating a alone feature because who were alone might have better chances to get off the boat\n",
    "    dataset['Alone'] = 0\n",
    "    dataset.loc[dataset['FamilySize'] == 1, 'Alone'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''in the description it is given that women and children had better survival chance so creating a feature that indicates\n",
    "about child'''\n",
    "for dataset in combine:\n",
    "    dataset['Child'] = 0\n",
    "    dataset.loc[(dataset['Age'] < 18) , 'Child'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Child  Survived\n",
      "1      1  0.539823\n",
      "0      0  0.361183\n",
      "--------------------\n",
      "   Alone  Survived\n",
      "0      0  0.505650\n",
      "1      1  0.303538\n",
      "--------------------\n",
      "   FamilySize  Survived\n",
      "3           4  0.724138\n",
      "2           3  0.578431\n",
      "1           2  0.552795\n",
      "6           7  0.333333\n",
      "0           1  0.303538\n",
      "4           5  0.200000\n",
      "5           6  0.136364\n",
      "7           8  0.000000\n",
      "8          11  0.000000\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "#now checking survival for new features\n",
    "cols_categorical = ['Child', 'Alone', 'FamilySize']\n",
    "for cols in cols_categorical:\n",
    "    print(training_raw[['Survived', cols]].groupby(cols, as_index = False)['Survived'].mean().sort_values(by = 'Survived',\n",
    "                                                                                                  ascending = False))\n",
    "    print('-'*20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<seaborn.axisgrid.FacetGrid at 0x28bc06b4390>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAADQCAYAAABStPXYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAEiNJREFUeJzt3W3QXHV5x/HvTyJYocqDgYmADbYZFGlViApSrRU7RbRCa2ih1kYHJ33hAz6NhvpCHadTmHG09EHGDKjRcZQHacmgI9IArXbaaFAEISqpUoigJFWw2I4avfrinMhNuGPu3Lv3vf/d/X5mdnbP2bNnr5zkym/P/5w9m6pCkqTWPGrUBUiSNBsDSpLUJANKktQkA0qS1CQDSpLUJANKktQkA2oEkrwjyW1Jbklyc5LnDGm9L0uydkjrenAI6zggyWVJtibZlGT54JVpkk1Rbzw/yZeT7Eyyahh1TaIloy5g2iQ5GXgpcEJV/TjJE4D99+H1S6pq52zPVdUGYMNwKh2Kc4EfVNVvJDkbuBD4kxHXpEZNWW/cBbwKeOuI62iae1CLbxmwo6p+DFBVO6rqHoAkd/ZNSZKVSW7sH78ryboknwM+2u+NPG3XCpPcmOTEJK9K8vdJHt+v61H9849NcneSRyf59SSfTXJTks8neUq/zDFJ/j3Jl5K8Z0h/1jOA9f3jK4FTk2RI69bkmZreqKo7q+oW4OfDWN+kMqAW3+eAo5N8M8kHkvzOHF93InBGVf0p8EngjwGSLAOeWFU37Vqwqh4AvgrsWvcfANdW1U+BdcDrq+pEuk9vH+iXuQi4uKqeBXx3T0X0jXvzLLcXzbL4kcDdfU07gQeAw+b459X0mabe0Bw4xLfIqurBJCcCzwN+F7gsydqq+sheXrqhqv6vf3w5cB3wTrpmvGKW5S+jG067ATgb+ECSg4DnAlfM2JE5oL8/BXh5//hjdMNxs9X/vL3UOdNse0teW0uzmrLe0BwYUCNQVT8DbgRuTHIrsBr4CLCTh/ZqH7Pby3404/XfSfLfSX6LrtH+Ypa32QD8dZJD6T5hXg8cCNxfVc/YU2l7qz3J54FfneWpt1bVP+82bxtwNLAtyRLg8cD39/Yeml5T1BuaA4f4FlmSY5OsmDHrGcB/9Y/vpGsYeOgT2558Engb8PiqunX3J6vqQeCLdMMT11TVz6rqh8C3k5zV15IkT+9f8m90nyYBXrGnN62q51XVM2a5zdaAG+j+gwFYBVxfXp1YezBlvaE5MKAW30HA+iS3J7kFOA54V//cu4GL+k9iP9vLeq6ka5rLf8kylwF/1t/v8grg3CRfBW6jO5EB4DzgtUm+RLenMwyXAocl2Qq8GRjKab6aWFPTG0melWQbcBbwwSS3DWO9kyZ+oJUktcg9KElSkwwoSVKTDChJUpMMKElSk5oIqNNOO63ovmfgzdsk3QZmb3ib0NucNBFQO3bsGHUJUpPsDU2zJgJKkqTdGVCSpCYZUJKkJhlQkqQmGVCSpCYZUJKkJvl7UCOwfO2nHzHvzgteMoJKJKld7kFJkpq014BK8qEk9yX52ox5hya5Lskd/f0h/fwk+dskW5PckuSEhSxekjS55rIH9RHgtN3mrQU2VtUKYCMP/RDdi4EV/W0NcPFwypQkTZu9BlRV/Svw/d1mnwGs7x+vB86cMf+j1fkP4OAky4ZVrCRpesz3GNQRVXUvQH9/eD//SODuGctt6+c9QpI1STYn2bx9+/Z5liFNHntD6gz7JInMMm/WK9dW1bqqWllVK5cuXTrkMqTxZW9InfkG1Pd2Dd319/f187cBR89Y7ijgnvmXJ0maVvMNqA3A6v7xauDqGfP/vD+b7yTggV1DgZIk7Yu9flE3ySeAFwBPSLINeCdwAXB5knOBu4Cz+sU/A5wObAX+F3j1AtQsSZoCew2oqjpnD0+dOsuyBbx20KIkSfJKEpKkJhlQkqQmGVCSpCYZUJKkJhlQkqQmGVCSpCYZUJKkJhlQkqQmGVCSpCYZUJKkJhlQkqQmGVCSpCYZUJKkJhlQkqQmGVCSpCYZUJKkJhlQkqQmGVCSpCYNFFBJ3pTktiRfS/KJJI9JckySTUnuSHJZkv2HVawkaXrMO6CSHAm8AVhZVccD+wFnAxcC76+qFcAPgHOHUagkaboMOsS3BPiVJEuAxwL3Ai8EruyfXw+cOeB7SJKm0LwDqqq+A7wXuIsumB4AbgLur6qd/WLbgCNne32SNUk2J9m8ffv2+ZYhTRx7Q+oMMsR3CHAGcAzwROBA4MWzLFqzvb6q1lXVyqpauXTp0vmWIU0ce0PqDDLE9yLg21W1vap+ClwFPBc4uB/yAzgKuGfAGiVJU2iQgLoLOCnJY5MEOBW4HbgBWNUvsxq4erASJUnTaJBjUJvoTob4MnBrv651wNuBNyfZChwGXDqEOiVJU2bJ3hfZs6p6J/DO3WZ/C3j2IOuVJMkrSUiSmmRASZKaZEBJkppkQEmSmmRASZKaZEBJkppkQEmSmmRASZKaZEBJkppkQEmSmmRASZKaZEBJkppkQEmSmmRASZKaZEBJkppkQEmSmmRASZKaZEBJkppkQEmSmjRQQCU5OMmVSb6eZEuSk5McmuS6JHf094cMq1hJ0vQYdA/qIuCzVfUU4OnAFmAtsLGqVgAb+2lJkvbJvAMqyeOA5wOXAlTVT6rqfuAMYH2/2HrgzEGLlCRNn0H2oJ4MbAc+nOQrSS5JciBwRFXdC9DfHz7bi5OsSbI5yebt27cPUIY0WewNqTNIQC0BTgAurqpnAj9iH4bzqmpdVa2sqpVLly4doAxpstgbUmeQgNoGbKuqTf30lXSB9b0kywD6+/sGK1GSNI3mHVBV9V3g7iTH9rNOBW4HNgCr+3mrgasHqlCSNJWWDPj61wMfT7I/8C3g1XShd3mSc4G7gLMGfA9J0hQaKKCq6mZg5SxPnTrIeiVNruVrP/2IeXde8JIRVKLWeSUJSVKTDChJUpMMKElSkwY9SUIzOLYuScPjHpQkqUkGlCSpSQaUJKlJBpQkqUkGlCSpSQaUJKlJBpQkqUl+D6oRfodKkh7OPShJUpMMKElSkwwoSVKTDChJUpM8SWKBzXbygyRp79yDkiQ1aeCASrJfkq8kuaafPibJpiR3JLksyf6DlylJmjbD2IM6D9gyY/pC4P1VtQL4AXDuEN5DkjRlBgqoJEcBLwEu6acDvBC4sl9kPXDmIO8hSZpOg+5B/Q3wNuDn/fRhwP1VtbOf3gYcOdsLk6xJsjnJ5u3btw9YhjQ57A2pM++ASvJS4L6qumnm7FkWrdleX1XrqmplVa1cunTpfMuQJo69IXUGOc38FOBlSU4HHgM8jm6P6uAkS/q9qKOAewYvU5I0bea9B1VV51fVUVW1HDgbuL6qXgHcAKzqF1sNXD1wlZKkqbMQ34N6O/DmJFvpjkldugDvIUmacEO5kkRV3Qjc2D/+FvDsYaxXkjS9vNTRFPE3pySNEwNqDvyPXZIWn9fikyQ1yYCSJDXJgJIkNcljUJIm1p5+j81jyOPBPShJUpMMKElSkwwoSVKTPAYlaax4XGl6GFDztKcmWej3sAk1TRajz9Quh/gkSU0yoCRJTXKIb8w47CdpWrgHJUlqkgElSWqSASVJapIBJUlq0rwDKsnRSW5IsiXJbUnO6+cfmuS6JHf094cMr1xJ0rQYZA9qJ/CWqnoqcBLw2iTHAWuBjVW1AtjYT0uStE/mfZp5Vd0L3Ns//p8kW4AjgTOAF/SLrQduBN4+UJWSJppXjNBshvI9qCTLgWcCm4Aj+vCiqu5NcvgeXrMGWAPwpCc9aRhlaEj8rtVo7Wtv+PelSTXwSRJJDgI+Bbyxqn4419dV1bqqWllVK5cuXTpoGdLEsDekzkABleTRdOH08aq6qp/9vSTL+ueXAfcNVqIkaRoNchZfgEuBLVX1vhlPbQBW949XA1fPvzxJ0rQa5BjUKcArgVuT3NzP+0vgAuDyJOcCdwFnDVaiJGkaDXIW3xeA7OHpU+e7Xu07D5KrRZ6Zp0F5NXNJU8cPdePBgJpQfnrVYvLfmxaC1+KTJDXJgJIkNckhPg3Esfzp0vJQ3qC17en1/nseHfegJElNMqAkSU1yiE+aQA5XaRKMVUB5vGP4Wj6mIGm6jVVASdJi25cPxn6IHi6PQUmSmjQ1e1B+spGk8TI1ASVJw+Kx28XhEJ8kqUnuQWlkHHbVNNiXvS3//T/cRAbUILvf7rrPbl+2yyDBM+zQMgQfzu2hcTKRAaX2LMaHBv+j1TSZhi9jewxKktSksd+DcjhPGox90I5p2CvaFwsSUElOAy4C9gMuqaoLFuJ9JGkaTOuHiKEHVJL9gH8Afg/YBnwpyYaqun3Y7yXNhycKSMOx0Ht8C3EM6tnA1qr6VlX9BPgkcMYCvI8kaYKlqoa7wmQVcFpVvaaffiXwnKp63W7LrQHW9JPHAt/YwyqfAOwYapELYxzqHIcaYTzqnEuNO6rqtH1d8T70xlzrGLVxqBHGo85xqBH2XuecemMhjkFllnmPSMGqWges2+vKks1VtXIYhS2kcahzHGqE8ahzIWuca28sdB3DMg41wnjUOQ41wvDqXIghvm3A0TOmjwLuWYD3kSRNsIUIqC8BK5Ick2R/4GxgwwK8jyRpgg19iK+qdiZ5HXAt3WnmH6qq2wZY5ZyGOhowDnWOQ40wHnW2UmMrdfwy41AjjEed41AjDKnOoZ8kIUnSMHipI0lSkwwoSVKTmg6oJKcl+UaSrUnWjroegCRHJ7khyZYktyU5r59/aJLrktzR3x8y6lqhu7JHkq8kuaafPibJpr7Oy/oTWUZZ38FJrkzy9X6bntzitkzypv7v+2tJPpHkMaPcli32BoxXf7TeG31NzffHQvZGswE145JJLwaOA85JctxoqwJgJ/CWqnoqcBLw2r6utcDGqloBbOynW3AesGXG9IXA+/s6fwCcO5KqHnIR8NmqegrwdLpam9qWSY4E3gCsrKrj6U7+OZsRbcuGewPGqz9a7w1ovD8WvDeqqskbcDJw7Yzp84HzR13XLHVeTXfdwW8Ay/p5y4BvNFDbUXT/gF8IXEP3JeodwJLZtvEI6nsc8G36k3VmzG9qWwJHAncDh9Kd+XoN8Puj2pbj0ht9bU32R+u90dfQfH8sdG80uwfFQ3/wXbb185qRZDnwTGATcERV3QvQ3x8+usp+4W+AtwE/76cPA+6vqp399Ki36ZOB7cCH+6GWS5IcSGPbsqq+A7wXuAu4F3gAuInRbcvmewOa74/WewPGoD8WujdaDqg5XTJpVJIcBHwKeGNV/XDU9ewuyUuB+6rqppmzZ1l0lNt0CXACcHFVPRP4EW0M/TxMP8Z/BnAM8ETgQLrhtd0t1rZs7e/xEVrujzHpDRiD/ljo3mg5oJq9ZFKSR9M138er6qp+9veSLOufXwbcN6r6eqcAL0tyJ90V5V9I96nx4CS7vqA96m26DdhWVZv66SvpGrK1bfki4NtVtb2qfgpcBTyX0W3LZnsDxqI/xqE3YDz6Y0F7o+WAavKSSUkCXApsqar3zXhqA7C6f7yabux9ZKrq/Ko6qqqW022766vqFcANwKp+sZHWWVXfBe5Ocmw/61TgdhrblnTDFycleWz/97+rzlFtyyZ7A8ajP8ahN2Bs+mNhe2OUBwHncADudOCbwH8C7xh1PX1Nv023u3oLcHN/O51uDHsjcEd/f+ioa51R8wuAa/rHTwa+CGwFrgAOGHFtzwA299vzn4BDWtyWwLuBrwNfAz4GHDDKbdlib/R1jVV/tNwbfU3N98dC9oaXOpIkNanlIT5J0hQzoCRJTTKgJElNMqAkSU0yoCRJTTKgJkySP0xSSZ4y6lqk1tgf48WAmjznAF+g+wKipIezP8aIATVB+uufnUJ3afuz+3mPSvKB/vdarknymSSr+udOTPIvSW5Kcu2uy6dIk8j+GD8G1GQ5k+63Y74JfD/JCcAfAcuB3wReQ3fp+13XS/s7YFVVnQh8CPirURQtLRL7Y8ws2fsiGiPn0F30ErqLYJ4DPBq4oqp+Dnw3yQ3988cCxwPXdZfQYj+6y+VLk8r+GDMG1IRIchjdVZmPT1J0DVXAP+7pJcBtVXXyIpUojYz9MZ4c4pscq4CPVtWvVdXyqjqa7tc4dwAv78faj6C7OCZ0v8q5NMkvhjSSPG0UhUuLwP4YQwbU5DiHR34a/BTdj4hto7vS8Afpft30gar6CV3TXpjkq3RXnX7u4pUrLSr7Ywx5NfMpkOSgqnqwH+b4InBKdb81I009+6NdHoOaDtckORjYH3iPzSc9jP3RKPegJElN8hiUJKlJBpQkqUkGlCSpSQaUJKlJBpQkqUn/D5UZlTWgN5GjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#it seeems like family size does not plat that much rule on determining\n",
    "#the survival rate so we will ignore that but for that we will keep it\n",
    "\n",
    "# now visualizing the numerical feature which are age and Fare\n",
    "g = sns.FacetGrid(training_raw, col = 'Survived')\n",
    "g.map(plt.hist, 'Age', bins = 30)\n",
    "\n",
    "#it seems that with little age and those with age high had high survival rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<seaborn.axisgrid.FacetGrid at 0x28bc083ef28>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAADQCAYAAABStPXYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAEspJREFUeJzt3X/QnWV95/H3p4BohUGQwKSQ2aCbdsWdGiFFLKtLl/6I9Ed0Flqs1bRDJ51d3NFtO93Q7qx2dpylO1tbnRZqujLGjgpYdcxQt0oRprZTgUBDIGTRtGQlJktCrfijrWPid/841wNnwwnPr/Pjfs55v2bOnPtc5z739X3ynCufc9/nfq47VYUkSV3zXZMuQJKkQQwoSVInGVCSpE4yoCRJnWRASZI6yYCSJHWSATUBSX4jyZ4ku5PsSvKqIW33p5JsHdK2vjGEbZya5NYk+5Lck2Tt8ivTNJuhsfHaJA8kOZrkqmHUNY1OnnQBsybJq4GfAC6qqm8lORt43iJef3JVHR30XFXtAHYMp9KhuBb4+6r650muAX4L+JkJ16SOmrGx8SXg54FfnXAdneYe1PitBp6sqm8BVNWTVXUQIMn+NihJsiHJ3W35nUm2JfkM8MG2N/LyuQ0muTvJxUl+PsnvJTmjbeu72vPfneTxJKckeWmSP01yf5LPJfkXbZ0LkvxVkvuS/Nch/aybgO1t+Y+BK5JkSNvW9JmZsVFV+6tqN/CdYWxvWhlQ4/cZYE2SLyS5Mcm/XuDrLgY2VdXPArcAPw2QZDXwPVV1/9yKVfUU8CAwt+2fBD5dVd8GtgH/oaoupvfp7ca2znuAm6rqB4D/e6Ii2sDdNeD2wwNWPw94vNV0FHgKePECf17NnlkaG1oAD/GNWVV9I8nFwGuAHwJuTbK1qj4wz0t3VNU/tuXbgDuAd9AbjB8dsP6t9A6n3QVcA9yY5DTgB4GP9u3InNruLwP+bVv+I3qH4wbV/5p56uw3aG/JubU00IyNDS2AATUBVXUMuBu4O8lDwGbgA8BRntmrff5xL/tm3+u/nOTvknw/vYH2SwO62QH8tyRn0fuE+VnghcBXq2r9iUqbr/YknwNOH/DUr1bVnx3XdgBYAxxIcjJwBvCV+frQ7JqhsaEF8BDfmCX5viTr+prWA/+nLe+nN2DgmU9sJ3IL8GvAGVX10PFPVtU3gHvpHZ64vaqOVdXXgMeSXN1qSZJXtJf8Jb1PkwBvOlGnVfWaqlo/4DZoAO6g9x8MwFXAZ8vZiXUCMzY2tAAG1PidBmxP8kiS3cCFwDvbc78JvKd9Ejs2z3b+mN6gue051rkV+Ll2P+dNwLVJHgT20DuRAeBtwHVJ7qO3pzMM7wdenGQf8MvAUE7z1dSambGR5AeSHACuBt6XZM8wtjtt4gdaSVIXuQclSeokA0qS1EkGlCSpkwwoSVIndSKgNm7cWPT+zsCbt2m6DYXjw9sU3hakEwH15JNPTroEqbMcH5pVnQgoSZKON29AJVmT5K4ke9O7TsvbWvs7k3y5b0LEK/tec3161wB6NMmPjfIHkCRNp4XMxXcU+JWqeiDJ6cD9Se5oz/1OVf2P/pWTXEjvr7hfDnwP8GdJvrfNsSVJ0oLMuwdVVYeq6oG2/HVgL73LKJzIJuCWqvpWVT0G7AMuGUaxkqTZsajvoNK7ZPcrgXta01vTuzTzzUnObG1PXwOoOcCAQEuyJcnOJDuPHDmy6MKlaeb4kBYRUO16KR8D3t5m/r0JeCm9GYcPAb89t+qAlz/rtMKq2lZVG6pqw6pVqxZduDTNHB/SAq8HleQUeuH0oar6OEBVPdH3/B8Ct7eHc9cAmnM+cHA5Ra7d+idPL++/4ceXsylJ0gqxkLP4Qu+yCXur6t197av7VnsD8HBb3gFck+TUJBcA6+hde0WSpAVbyB7UZcCbgYeS7Gptvw68Mcl6eofv9tOuXFlVe5LcBjxC7wzA6zyDT5K0WPMGVFX9BYO/V/rUc7zmXcC7llGXJGnGOZOEJKmTDChJUicZUJKkTjKgJEmdZEBJkjrJgJIkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROMqAkSZ1kQEmSOsmAkiR1kgElSeokA0qS1EkGlCSpkwwoSVInGVCSpE4yoCRJnTRvQCVZk+SuJHuT7EnyttZ+VpI7knyx3Z/Z2pPkvUn2Jdmd5KJR/xCSpOmzkD2oo8CvVNXLgEuB65JcCGwF7qyqdcCd7THA64B17bYFuGnoVUuSpt68AVVVh6rqgbb8dWAvcB6wCdjeVtsOvL4tbwI+WD2fB16UZPXQK5ckTbVFfQeVZC3wSuAe4NyqOgS9EAPOaaudBzze97IDre34bW1JsjPJziNHjiy+cmmKOT6kRQRUktOAjwFvr6qvPdeqA9rqWQ1V26pqQ1VtWLVq1ULLkGaC40NaYEAlOYVeOH2oqj7emp+YO3TX7g+39gPAmr6Xnw8cHE65kqRZsZCz+AK8H9hbVe/ue2oHsLktbwY+2df+lnY236XAU3OHAiVJWqiTF7DOZcCbgYeS7Gptvw7cANyW5FrgS8DV7blPAVcC+4B/AH5hqBVLkmbCvAFVVX/B4O+VAK4YsH4B1y2zLknSjHMmCUlSJxlQkqROMqAkSZ1kQEmSOsmAkiR1kgElSeokA0qS1EkGlCSpkwwoSVInGVCSpE4yoCRJnWRASZI6yYCSJHWSASVJ6iQDSpLUSQaUJKmTDChJUicZUJKkTjKgJEmdZEBJkjpp3oBKcnOSw0ke7mt7Z5IvJ9nVblf2PXd9kn1JHk3yY6MqXJI03RayB/UBYOOA9t+pqvXt9imAJBcC1wAvb6+5MclJwypWkjQ75g2oqvpz4CsL3N4m4Jaq+lZVPQbsAy5ZRn2SpBm1nO+g3ppkdzsEeGZrOw94vG+dA63tWZJsSbIzyc4jR44sowxp+jg+pKUH1E3AS4H1wCHgt1t7BqxbgzZQVduqakNVbVi1atUSy5Cmk+NDWmJAVdUTVXWsqr4D/CHPHMY7AKzpW/V84ODySpQkzaIlBVSS1X0P3wDMneG3A7gmyalJLgDWAfcur0RJ0iw6eb4VknwEuBw4O8kB4B3A5UnW0zt8tx/4JYCq2pPkNuAR4ChwXVUdG03pkqRpNm9AVdUbBzS//znWfxfwruUUJUmSM0lIkjrJgJIkdZIBJUnqJANKktRJ854kIanb1m79k2e17b/hxydQiTRc7kFJkjrJgJIkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROMqAkSZ204v5Qt/+PEv1jREmaXu5BSZI6yYCSJHWSASVJ6iQDSpLUSQaUJKmT5g2oJDcnOZzk4b62s5LckeSL7f7M1p4k702yL8nuJBeNsnhJ0vRayB7UB4CNx7VtBe6sqnXAne0xwOuAde22BbhpOGVKkmbNvAFVVX8OfOW45k3A9ra8HXh9X/sHq+fzwIuSrB5WsZKk2bHU76DOrapDAO3+nNZ+HvB433oHWtuzJNmSZGeSnUeOHFliGdJ0cnxIwz9JIgPaatCKVbWtqjZU1YZVq1YNuQxpZXN8SEsPqCfmDt21+8Ot/QCwpm+984GDSy9PkjSrlhpQO4DNbXkz8Mm+9re0s/kuBZ6aOxQoSdJizDtZbJKPAJcDZyc5ALwDuAG4Lcm1wJeAq9vqnwKuBPYB/wD8wghqliTNgHkDqqreeIKnrhiwbgHXLbcoSZKcSUKS1EkGlCSpk1bcBQslza//wp79vMinVhL3oCRJnWRASZI6yYCSJHWSASVJ6iQDSpLUSQaUJKmTDChJUicZUJKkTjKgJEmdZEBJkjrJgJIkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROWtEXLOy/KJsXYpOk6bKsgEqyH/g6cAw4WlUbkpwF3AqsBfYDP11Vf7+8MiVJs2YYe1A/VFVP9j3eCtxZVTck2doe/6ch9POc3JuSpOkyiu+gNgHb2/J24PUj6EOSNOWWG1AFfCbJ/Um2tLZzq+oQQLs/Z9ALk2xJsjPJziNHjiyzDGm6OD6k5QfUZVV1EfA64Lokr13oC6tqW1VtqKoNq1atWmYZ0nRxfEjLDKiqOtjuDwOfAC4BnkiyGqDdH15ukZKk2bPkgErywiSnzy0DPwo8DOwANrfVNgOfXG6RkqTZs5yz+M4FPpFkbjsfrqo/TXIfcFuSa4EvAVcvv0xJ0qxZckBV1d8CrxjQ/nfAFcspSpIkpzqSJHWSASVJ6iQDSpLUSQaUJKmTDChJUiet6MttSFq+/omW+znpsibNPShJUie5ByVpoEF7Vu5VaZzcg5IkdZIBJUnqJANKktRJBpQkqZMMKElSJxlQkqROmsrTzD09VpJWvqkMKEmDnWjWCKmLDChJC+a0SBqnmQyo/kHmwJJGwzDTcs1kQPUzrCSpm2YmoDz2Lo3OYsaXJzFpoUYWUEk2Au8BTgL+Z1XdMKq+hsW9KUnqjpEEVJKTgN8HfgQ4ANyXZEdVPTKK/sZlqQFm8Emjt5i9uEHj0O/MFm/U/2aj2oO6BNhXVX8LkOQWYBOwYgKqy2G02DeFh1TUdeMOBw/5rwypquFvNLkK2FhVv9gevxl4VVW9tW+dLcCW9vD7gEefY5NnA08OvdDFmXQN9r/y3gNPVtXGpXS0wsaH/a+89+ak+1/Q2BjVHlQGtP1/SVhV24BtC9pYsrOqNgyjsKWadA32P1vvgZU0Pux/tt6b4+x/VHPxHQDW9D0+Hzg4or4kSVNoVAF1H7AuyQVJngdcA+wYUV+SpCk0kkN8VXU0yVuBT9M7zfzmqtqzjE0u6FDHiE26BvufvC7UMMik67L/yZt0DSPpfyQnSUiStFxeD0qS1EkGlCSpkzofUEk2Jnk0yb4kW0fUx81JDid5uK/trCR3JPliuz+ztSfJe1s9u5NcNIT+1yS5K8neJHuSvG2cNSR5fpJ7kzzY+v/N1n5Bknta/7e2E15Icmp7vK89v3Z5/wJP13FSkr9OcvuE+t+f5KEku5LsbG1jex8sod6Rj43Wj+NjxsfHxMZGVXX2Ru8Ei78BXgI8D3gQuHAE/bwWuAh4uK/tvwNb2/JW4Lfa8pXA/6L3t16XAvcMof/VwEVt+XTgC8CF46qhbee0tnwKcE/b7m3ANa39D4B/15b/PfAHbfka4NYh/R5+GfgwcHt7PO7+9wNnH9c2tvfBImsdy9hofTk+Znx8TGpsjG1ALfEf5dXAp/seXw9cP6K+1h43AB8FVrfl1cCjbfl9wBsHrTfEWj5Jbx7DsdcAfDfwAPAqen8ZfvLxvwt6Z2e+ui2f3NbLMvs9H7gT+DfA7e3NPbb+27YGDcKJvQ/mqXVsY6Nt3/FRszs+JjU2un6I7zzg8b7HB1rbOJxbVYcA2v0546ip7Y6/kt6ntLHV0A4f7AIOA3fQ+3T+1ao6OqCPp/tvzz8FvHg5/QO/C/wa8J32+MVj7h96s518Jsn96U01BBN6HyzApPt3fMzW+JjI2Oj69aDmnTJpAkZWU5LTgI8Bb6+qryWDuhpNDVV1DFif5EXAJ4CXPUcfQ+0/yU8Ah6vq/iSXL6CPUf0OLquqg0nOAe5I8r+fY91Jvzcn3f+JOD6mc3xMZGx0fQ9qklMmPZFkNUC7PzzKmpKcQm/wfaiqPj6JGgCq6qvA3fSOHb8oydyHmP4+nu6/PX8G8JVldHsZ8FNJ9gO30DuM8btj7B+AqjrY7g/T+0/oEibwO1igSffv+Jih8TGpsdH1gJrklEk7gM1teTO9495z7W9pZ6pcCjw1t5u7VOl9FHw/sLeq3j3uGpKsap8MSfIC4IeBvcBdwFUn6H+urquAz1Y72LwUVXV9VZ1fVWvp/Y4/W1VvGlf/AElemOT0uWXgR4GHGeP7YJEmPZ2Y42NGxsdEx8Zyvjgbx43eGSFfoHfM9zdG1MdHgEPAt+ml/7X0jtneCXyx3Z/V1g29izH+DfAQsGEI/f8rervAu4Fd7XbluGoAvh/469b/w8B/ae0vAe4F9gEfBU5t7c9vj/e1518yxN/F5TxzltLY+m99Pdhue+bea+N8H3RxbDg+HB+THBtOdSRJ6qSuH+KTJM0oA0qS1EkGlCSpkwwoSVInGVCSpE7q+kwSWqAkx+id0jnn9VW1f0LlSJ3h2Fi5PM18SiT5RlWdtoTXnVS9aVykqeTYWLk8xDfFkqxN8rkkD7TbD7b2y9O7vs6HaZ8sk/xcete82ZXkfUlOmmjx0gg5NlYGD/FNjxe02ZYBHquqN9CbG+tHquqfkqyjNyPAhrbOJcC/rKrHkrwM+Bl6E0J+O8mNwJuAD475Z5BGwbGxQhlQ0+Mfq2r9cW2nAL+XZD1wDPjevufurarH2vIVwMXAfW2G6BfwzMSP0krn2FihDKjp9h+BJ4BX0Duc+099z32zbznA9qq6foy1SZPk2FgB/A5qup0BHKqq7wBvpneZ8EHuBK5q13ohyVlJ/tmYapQmwbGxAhhQ0+1GYHOSz9M7hPHNQStV1SPAf6Z3xczd9K4YunpsVUrj59hYATzNXJLUSe5BSZI6yYCSJHWSASVJ6iQDSpLUSQaUJKmTDChJUicZUJKkTvp/N+C1BGb4FUIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Similarly for fare\n",
    "g = sns.FacetGrid(training_raw, col = 'Survived')\n",
    "g.map(plt.hist, 'Fare', bins = 30)\n",
    "# it seems that those with high fare had high survival chance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>FamilySize</th>\n",
       "      <th>Alone</th>\n",
       "      <th>Child</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>S</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>S</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>S</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>S</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name  Sex   Age     Fare  \\\n",
       "0                            Braund, Mr. Owen Harris    0  22.0   7.2500   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...    1  38.0  71.2833   \n",
       "2                             Heikkinen, Miss. Laina    1  26.0   7.9250   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)    1  35.0  53.1000   \n",
       "4                           Allen, Mr. William Henry    0  35.0   8.0500   \n",
       "\n",
       "  Embarked  FamilySize  Alone  Child  \n",
       "0        S           2      0      0  \n",
       "1        C           2      0      0  \n",
       "2        S           1      1      0  \n",
       "3        S           2      0      0  \n",
       "4        S           1      1      0  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now handling name and Embarked variable\n",
    "for dataset in combine:\n",
    "    dataset['Title'] = dataset['Name'].str.split(',', expand = True)[1].str.split('.', expand = True)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " Mr        517\n",
       " Miss      182\n",
       " Mrs       125\n",
       " Master     40\n",
       "Misc        27\n",
       "Name: Title, dtype: int64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title_count = training_raw['Title'].value_counts() < 10\n",
    "training_raw['Title'] = training_raw['Title'].apply(lambda x: 'Misc' if title_count.loc[x] == True else str(x))\n",
    "training_raw['Title'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " Mr        240\n",
       " Miss       78\n",
       " Mrs        72\n",
       " Master     21\n",
       "Misc         7\n",
       "Name: Title, dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SImilarly for testing data\n",
    "title_count = testing_raw['Title'].value_counts() < 10\n",
    "testing_raw['Title'] = testing_raw['Title'].apply(lambda x: 'Misc' if title_count.loc[x] == True else str(x))\n",
    "testing_raw['Title'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dropping the name variable \n",
    "passengerId = testing_raw['PassengerId']\n",
    "for dataset in combine:\n",
    "    dataset.drop(['Name', 'PassengerId'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#handling categorical\n",
    "training_one_hot_encoded = pd.get_dummies(training_raw)\n",
    "testing_one_hot_encoded = pd.get_dummies(testing_raw)\n",
    "training_target = training_one_hot_encoded['Survived']\n",
    "training_one_hot_encoded.drop(['Survived'], axis = 1, inplace = True)\n",
    "training_predictors, testing_predictors = training_one_hot_encoded.align(testing_one_hot_encoded, \n",
    "                                                                        join = 'left',\n",
    "                                                                        axis = 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>FamilySize</th>\n",
       "      <th>Alone</th>\n",
       "      <th>Child</th>\n",
       "      <th>Embarked_C</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>Embarked_S</th>\n",
       "      <th>Title_ Master</th>\n",
       "      <th>Title_ Miss</th>\n",
       "      <th>Title_ Mr</th>\n",
       "      <th>Title_ Mrs</th>\n",
       "      <th>Title_Misc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pclass  Sex   Age     Fare  FamilySize  Alone  Child  Embarked_C  \\\n",
       "0       3    0  22.0   7.2500           2      0      0           0   \n",
       "1       1    1  38.0  71.2833           2      0      0           1   \n",
       "2       3    1  26.0   7.9250           1      1      0           0   \n",
       "3       1    1  35.0  53.1000           2      0      0           0   \n",
       "4       3    0  35.0   8.0500           1      1      0           0   \n",
       "\n",
       "   Embarked_Q  Embarked_S  Title_ Master  Title_ Miss  Title_ Mr  Title_ Mrs  \\\n",
       "0           0           1              0            0          1           0   \n",
       "1           0           0              0            0          0           1   \n",
       "2           0           1              0            1          0           0   \n",
       "3           0           1              0            0          0           1   \n",
       "4           0           1              0            0          1           0   \n",
       "\n",
       "   Title_Misc  \n",
       "0           0  \n",
       "1           0  \n",
       "2           0  \n",
       "3           0  \n",
       "4           0  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_predictors.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8327803314039268"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = training_predictors.astype('float')\n",
    "estimators = [('std', StandardScaler()), ('Forest',RandomForestClassifier(n_estimators=100, max_depth=10, max_leaf_nodes=11,\n",
    "                                                                         random_state = 0))]\n",
    "pipeline = Pipeline(estimators)\n",
    "results = cross_val_score(pipeline, X, training_target, cv = 10)\n",
    "results.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "Age_mean = training_predictors['Age'].mean()\n",
    "Age_std = training_predictors['Age'].std()\n",
    "Fare_mean = training_predictors['Fare'].mean()\n",
    "Fare_std = training_predictors['Fare'].std()\n",
    "combine = [training_predictors, testing_predictors]\n",
    "for dataset in combine:\n",
    "    dataset['Age'] = (dataset['Age']- Age_mean)/Age_std\n",
    "    dataset['Fare'] = (dataset['Fare'] - Fare_mean)/ Fare_std\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>FamilySize</th>\n",
       "      <th>Alone</th>\n",
       "      <th>Child</th>\n",
       "      <th>Embarked_C</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>Embarked_S</th>\n",
       "      <th>Title_ Master</th>\n",
       "      <th>Title_ Miss</th>\n",
       "      <th>Title_ Mr</th>\n",
       "      <th>Title_ Mrs</th>\n",
       "      <th>Title_Misc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.548625</td>\n",
       "      <td>-0.502163</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.654975</td>\n",
       "      <td>0.786404</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.247725</td>\n",
       "      <td>-0.488580</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.429300</td>\n",
       "      <td>0.420494</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.429300</td>\n",
       "      <td>-0.486064</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pclass  Sex       Age      Fare  FamilySize  Alone  Child  Embarked_C  \\\n",
       "0       3    0 -0.548625 -0.502163           2      0      0           0   \n",
       "1       1    1  0.654975  0.786404           2      0      0           1   \n",
       "2       3    1 -0.247725 -0.488580           1      1      0           0   \n",
       "3       1    1  0.429300  0.420494           2      0      0           0   \n",
       "4       3    0  0.429300 -0.486064           1      1      0           0   \n",
       "\n",
       "   Embarked_Q  Embarked_S  Title_ Master  Title_ Miss  Title_ Mr  Title_ Mrs  \\\n",
       "0           0           1              0            0          1           0   \n",
       "1           0           0              0            0          0           1   \n",
       "2           0           1              0            1          0           0   \n",
       "3           0           1              0            0          0           1   \n",
       "4           0           1              0            0          1           0   \n",
       "\n",
       "   Title_Misc  \n",
       "0           0  \n",
       "1           0  \n",
       "2           0  \n",
       "3           0  \n",
       "4           0  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_predictors.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RandomForestClassifier(n_estimators=100, max_depth=10, max_leaf_nodes=11,\n",
    "                                                                         random_state = 0)\n",
    "model.fit(training_predictors, training_target)\n",
    "answers = model.predict(testing_predictors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_file = pd.DataFrame({'PassengerId':passengerId, 'Survived':answers})\n",
    "final_file.to_csv('RandomResult.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-error:0.201493\n",
      "Will train until validation_0-error hasn't improved in 5 rounds.\n",
      "[1]\tvalidation_0-error:0.19403\n",
      "[2]\tvalidation_0-error:0.186567\n",
      "[3]\tvalidation_0-error:0.186567\n",
      "[4]\tvalidation_0-error:0.186567\n",
      "[5]\tvalidation_0-error:0.171642\n",
      "[6]\tvalidation_0-error:0.171642\n",
      "[7]\tvalidation_0-error:0.16791\n",
      "[8]\tvalidation_0-error:0.164179\n",
      "[9]\tvalidation_0-error:0.164179\n",
      "[10]\tvalidation_0-error:0.156716\n",
      "[11]\tvalidation_0-error:0.156716\n",
      "[12]\tvalidation_0-error:0.156716\n",
      "[13]\tvalidation_0-error:0.156716\n",
      "[14]\tvalidation_0-error:0.156716\n",
      "[15]\tvalidation_0-error:0.145522\n",
      "[16]\tvalidation_0-error:0.145522\n",
      "[17]\tvalidation_0-error:0.149254\n",
      "[18]\tvalidation_0-error:0.145522\n",
      "[19]\tvalidation_0-error:0.149254\n",
      "[20]\tvalidation_0-error:0.152985\n",
      "Stopping. Best iteration:\n",
      "[15]\tvalidation_0-error:0.145522\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.04, max_delta_step=0,\n",
       "       max_depth=10, min_child_weight=1, missing=None, n_estimators=1000,\n",
       "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Using XGB Classifier\n",
    "train_X, test_X, train_y, test_y = train_test_split(training_predictors, training_target, test_size = 0.30, random_state = 0,\n",
    "                                                    stratify =training_target )\n",
    "my_model = XGBClassifier(n_estimators=1000, learning_rate=0.04, random_state = 0, max_depth=10)\n",
    "my_model.fit(train_X, train_y, early_stopping_rounds=5, \n",
    "             eval_set=[(test_X, test_y)], verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_model = XGBClassifier(n_estimators=15, learning_rate=0.04, random_state = 0, max_depth=10)\n",
    "my_model.fit(training_predictors, training_target)\n",
    "answers = my_model.predict(testing_predictors)\n",
    "final_file = pd.DataFrame({'PassengerId':passengerId, 'Survived':answers})\n",
    "final_file.to_csv('XGBResult.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_predictors.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# baseline model\n",
    "def create_baseline():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],))) #input shape need to be vector so here its (60, )\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = 'adam',loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.82% (3.80%)\n"
     ]
    }
   ],
   "source": [
    "seed = 7\n",
    "# evaluate model with standardized dataset\n",
    "estimator = KerasClassifier(build_fn=create_baseline, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.38% (3.87%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(15,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = 'adam',loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 81.93% (3.95%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(15,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = 'rmsprop',loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.26% (3.70%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(20,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = 'adam',loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.71% (4.00%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(70,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = 'adam',loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.49% (4.07%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(35,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = 'adam',loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 83.05% (3.08%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = 'adam',loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.27% (2.76%)\n"
     ]
    }
   ],
   "source": [
    "from keras import optimizers\n",
    "sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = sgd,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.49% (2.83%)\n"
     ]
    }
   ],
   "source": [
    "from keras import optimizers\n",
    "sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = sgd,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 74.77% (7.06%)\n"
     ]
    }
   ],
   "source": [
    "from keras import optimizers\n",
    "sgd = optimizers.SGD(lr=0.05, decay=1e-6, momentum=0.9)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = sgd,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 81.48% (2.89%)\n"
     ]
    }
   ],
   "source": [
    "from keras import optimizers\n",
    "sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.5)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = sgd,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 81.82% (3.18%)\n"
     ]
    }
   ],
   "source": [
    "from keras import optimizers\n",
    "sgd = optimizers.SGD(lr=0.01, momentum=0.9)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = sgd,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.16% (3.33%)\n"
     ]
    }
   ],
   "source": [
    "from keras import optimizers\n",
    "sgd = optimizers.SGD(lr=0.009, decay=1e-6, momentum=0.9)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = sgd,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 78.91% (6.15%)\n"
     ]
    }
   ],
   "source": [
    "from keras import optimizers\n",
    "Rms = optimizers.RMSprop(lr=0.05)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = Rms,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 81.04% (3.55%)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "Rms = optimizers.RMSprop(lr=0.009)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = Rms,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.94% (2.87%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "Adam = optimizers.Adam(lr=0.0009)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = Adam,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.94% (2.65%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "Adam = optimizers.Adam(lr=0.002)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = Adam,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.72% (2.24%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "Adam = optimizers.Adam(lr=0.003)\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = Adam,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 81.71% (2.78%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "np.random.seed(seed)\n",
    "def create_smaller():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10, activation = 'relu'))\n",
    "  model.add(layers.Dense(3, activation = 'relu'))\n",
    "  model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "  model.compile(optimizer = 'adam',loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 83.16% (3.35%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "seed = 7\n",
    "np.random.seed(seed)\n",
    "def create_smaller_1():\n",
    "  model = models.Sequential()\n",
    "  model.add(layers.Dense(60,activation = 'relu',kernel_initializer = 'normal', input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10,kernel_initializer = 'normal', activation = 'relu'))\n",
    "  model.add(layers.Dense(1,kernel_initializer = 'normal', activation = 'sigmoid'))\n",
    "  model.compile(optimizer = 'adam',loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller_1, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 61.50% (8.80%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "seed = 7\n",
    "np.random.seed(seed)\n",
    "def create_smaller_2():\n",
    "  sgd = SGD(lr=0.09, momentum=0.9, decay=0.0, nesterov=False)\n",
    "  model = models.Sequential()\n",
    "  \n",
    "  model.add(layers.Dense(60,activation = 'relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), \n",
    "                         input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "  model.add(layers.Dense(1,kernel_initializer = 'normal', activation = 'sigmoid'))\n",
    "  model.compile(optimizer = sgd,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller_2, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 61.62% (0.28%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "seed = 7\n",
    "np.random.seed(seed)\n",
    "def create_smaller_2():\n",
    "  sgd = SGD(lr=0.09, momentum=0.9, decay=0.0, nesterov=False)\n",
    "  model = models.Sequential()\n",
    "  \n",
    "  model.add(layers.Dense(60,activation = 'relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), \n",
    "                         input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dropout(0.20))\n",
    "  model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "  model.add(layers.Dropout(0.20))\n",
    "  model.add(layers.Dense(1,kernel_initializer = 'normal', activation = 'sigmoid'))\n",
    "  model.compile(optimizer = sgd,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller_2, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results: 82.15% (3.50%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "seed = 7\n",
    "np.random.seed(seed)\n",
    "def create_smaller_2():\n",
    "  model = models.Sequential()\n",
    "  adam = optimizers.Adam(lr = 0.002)\n",
    "  model.add(layers.Dense(60,activation = 'relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), \n",
    "                         input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dropout(0.20))\n",
    "  model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "  model.add(layers.Dropout(0.20))\n",
    "  model.add(layers.Dense(1,kernel_initializer = 'normal', activation = 'sigmoid'))\n",
    "  model.compile(optimizer = adam,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller_2, epochs=9, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "801/801 [==============================] - 3s 4ms/step - loss: 0.5999 - acc: 0.6879\n",
      "Epoch 2/20\n",
      "801/801 [==============================] - 0s 395us/step - loss: 0.4677 - acc: 0.8027\n",
      "Epoch 3/20\n",
      "801/801 [==============================] - ETA: 0s - loss: 0.4823 - acc: 0.812 - 0s 429us/step - loss: 0.4690 - acc: 0.8202\n",
      "Epoch 4/20\n",
      "801/801 [==============================] - ETA: 0s - loss: 0.4476 - acc: 0.812 - 0s 504us/step - loss: 0.4451 - acc: 0.8177\n",
      "Epoch 5/20\n",
      "801/801 [==============================] - 0s 420us/step - loss: 0.4279 - acc: 0.8290\n",
      "Epoch 6/20\n",
      "801/801 [==============================] - 0s 409us/step - loss: 0.4375 - acc: 0.8165\n",
      "Epoch 7/20\n",
      "801/801 [==============================] - 0s 415us/step - loss: 0.4207 - acc: 0.8165\n",
      "Epoch 8/20\n",
      "801/801 [==============================] - 0s 489us/step - loss: 0.4304 - acc: 0.8252\n",
      "Epoch 9/20\n",
      "801/801 [==============================] - 0s 419us/step - loss: 0.4286 - acc: 0.8177\n",
      "Epoch 10/20\n",
      "801/801 [==============================] - 0s 425us/step - loss: 0.4209 - acc: 0.8302\n",
      "Epoch 11/20\n",
      "801/801 [==============================] - 0s 514us/step - loss: 0.4214 - acc: 0.8227\n",
      "Epoch 12/20\n",
      "801/801 [==============================] - 0s 404us/step - loss: 0.4200 - acc: 0.8227\n",
      "Epoch 13/20\n",
      "801/801 [==============================] - 0s 435us/step - loss: 0.4112 - acc: 0.8290\n",
      "Epoch 14/20\n",
      "801/801 [==============================] - 0s 469us/step - loss: 0.4155 - acc: 0.8327\n",
      "Epoch 15/20\n",
      "801/801 [==============================] - 0s 459us/step - loss: 0.4066 - acc: 0.8327\n",
      "Epoch 16/20\n",
      "801/801 [==============================] - 0s 439us/step - loss: 0.4211 - acc: 0.8315\n",
      "Epoch 17/20\n",
      "801/801 [==============================] - 0s 514us/step - loss: 0.4103 - acc: 0.8290\n",
      "Epoch 18/20\n",
      "801/801 [==============================] - 0s 405us/step - loss: 0.4047 - acc: 0.8352\n",
      "Epoch 19/20\n",
      "801/801 [==============================] - 0s 404us/step - loss: 0.4045 - acc: 0.8365\n",
      "Epoch 20/20\n",
      "801/801 [==============================] - 0s 419us/step - loss: 0.4119 - acc: 0.8265\n",
      "90/90 [==============================] - 1s 13ms/step\n",
      "Epoch 1/20\n",
      "801/801 [==============================] - 3s 4ms/step - loss: 0.6177 - acc: 0.6479\n",
      "Epoch 2/20\n",
      "801/801 [==============================] - 0s 419us/step - loss: 0.4845 - acc: 0.7903\n",
      "Epoch 3/20\n",
      "801/801 [==============================] - 0s 420us/step - loss: 0.4801 - acc: 0.7978\n",
      "Epoch 4/20\n",
      "801/801 [==============================] - 0s 499us/step - loss: 0.4694 - acc: 0.8115\n",
      "Epoch 5/20\n",
      "801/801 [==============================] - 0s 424us/step - loss: 0.4539 - acc: 0.8090\n",
      "Epoch 6/20\n",
      "801/801 [==============================] - 0s 424us/step - loss: 0.4425 - acc: 0.8177\n",
      "Epoch 7/20\n",
      "801/801 [==============================] - 0s 424us/step - loss: 0.4487 - acc: 0.8065\n",
      "Epoch 8/20\n",
      "801/801 [==============================] - 0s 424us/step - loss: 0.4410 - acc: 0.8152\n",
      "Epoch 9/20\n",
      "801/801 [==============================] - 0s 429us/step - loss: 0.4391 - acc: 0.8265 0s - loss: 0.4235 - acc: 0\n",
      "Epoch 10/20\n",
      "801/801 [==============================] - 0s 429us/step - loss: 0.4276 - acc: 0.8277\n",
      "Epoch 11/20\n",
      "801/801 [==============================] - 0s 434us/step - loss: 0.4273 - acc: 0.8290\n",
      "Epoch 12/20\n",
      "801/801 [==============================] - 0s 430us/step - loss: 0.4142 - acc: 0.8277 0s - loss: 0.4064 - acc: 0.\n",
      "Epoch 13/20\n",
      "801/801 [==============================] - 0s 414us/step - loss: 0.4215 - acc: 0.8277\n",
      "Epoch 14/20\n",
      "801/801 [==============================] - 0s 429us/step - loss: 0.4332 - acc: 0.8327\n",
      "Epoch 15/20\n",
      "801/801 [==============================] - 0s 434us/step - loss: 0.4285 - acc: 0.8327\n",
      "Epoch 16/20\n",
      "801/801 [==============================] - 0s 409us/step - loss: 0.4119 - acc: 0.8365\n",
      "Epoch 17/20\n",
      "801/801 [==============================] - 0s 434us/step - loss: 0.4156 - acc: 0.8390\n",
      "Epoch 18/20\n",
      "801/801 [==============================] - 0s 474us/step - loss: 0.4272 - acc: 0.8315\n",
      "Epoch 19/20\n",
      "801/801 [==============================] - 0s 554us/step - loss: 0.4120 - acc: 0.8414\n",
      "Epoch 20/20\n",
      "801/801 [==============================] - 0s 494us/step - loss: 0.4099 - acc: 0.8402\n",
      "90/90 [==============================] - 1s 15ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 3s 4ms/step - loss: 0.6047 - acc: 0.6696\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 514us/step - loss: 0.5020 - acc: 0.7905\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 584us/step - loss: 0.4758 - acc: 0.7943\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 504us/step - loss: 0.4528 - acc: 0.8130\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4612 - acc: 0.8242\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4397 - acc: 0.8217\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 504us/step - loss: 0.4475 - acc: 0.8142\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 494us/step - loss: 0.4397 - acc: 0.8180\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 594us/step - loss: 0.4367 - acc: 0.8204\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 499us/step - loss: 0.4322 - acc: 0.8142\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 1s 648us/step - loss: 0.4217 - acc: 0.8367\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 529us/step - loss: 0.4353 - acc: 0.8229\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 569us/step - loss: 0.4242 - acc: 0.8242\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 579us/step - loss: 0.4083 - acc: 0.8304\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 564us/step - loss: 0.4185 - acc: 0.8267\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 589us/step - loss: 0.4210 - acc: 0.8267\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 574us/step - loss: 0.4161 - acc: 0.8229\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 574us/step - loss: 0.4125 - acc: 0.8304\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 603us/step - loss: 0.4176 - acc: 0.8329\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 583us/step - loss: 0.4129 - acc: 0.8279\n",
      "89/89 [==============================] - 1s 15ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 3s 4ms/step - loss: 0.6129 - acc: 0.6708\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 504us/step - loss: 0.4771 - acc: 0.8080\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 1s 633us/step - loss: 0.4602 - acc: 0.8067\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 623us/step - loss: 0.4558 - acc: 0.8242\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 1s 658us/step - loss: 0.4429 - acc: 0.8254\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 584us/step - loss: 0.4357 - acc: 0.8217\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 579us/step - loss: 0.4383 - acc: 0.8342\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 574us/step - loss: 0.4339 - acc: 0.8279\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 1s 683us/step - loss: 0.4346 - acc: 0.8379\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 609us/step - loss: 0.4289 - acc: 0.8279\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 1s 633us/step - loss: 0.4158 - acc: 0.8367\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 1s 633us/step - loss: 0.4187 - acc: 0.8304\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 589us/step - loss: 0.4339 - acc: 0.8342\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 609us/step - loss: 0.4231 - acc: 0.8342\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 623us/step - loss: 0.4049 - acc: 0.8342\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 579us/step - loss: 0.4016 - acc: 0.8392\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 1s 643us/step - loss: 0.4152 - acc: 0.8416\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 584us/step - loss: 0.4053 - acc: 0.8441\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 603us/step - loss: 0.4235 - acc: 0.8354\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 529us/step - loss: 0.4148 - acc: 0.8379\n",
      "89/89 [==============================] - 1s 15ms/step\n",
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "802/802 [==============================] - 3s 4ms/step - loss: 0.5951 - acc: 0.6758\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 469us/step - loss: 0.4868 - acc: 0.7993\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 479us/step - loss: 0.4610 - acc: 0.8267\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 474us/step - loss: 0.4499 - acc: 0.8267\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 479us/step - loss: 0.4340 - acc: 0.8392\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 474us/step - loss: 0.4431 - acc: 0.8392\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 484us/step - loss: 0.4317 - acc: 0.8229\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4257 - acc: 0.8292\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 479us/step - loss: 0.4198 - acc: 0.8392\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4219 - acc: 0.8342\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 504us/step - loss: 0.4092 - acc: 0.8379\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4104 - acc: 0.8342\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 484us/step - loss: 0.4096 - acc: 0.8404\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 499us/step - loss: 0.4030 - acc: 0.8441\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 514us/step - loss: 0.4061 - acc: 0.8367\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 459us/step - loss: 0.3948 - acc: 0.8541\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 474us/step - loss: 0.4020 - acc: 0.8466\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 479us/step - loss: 0.3950 - acc: 0.8466\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 514us/step - loss: 0.3914 - acc: 0.8416\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 479us/step - loss: 0.4010 - acc: 0.8404\n",
      "89/89 [==============================] - 1s 14ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 3s 4ms/step - loss: 0.6098 - acc: 0.6584\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 474us/step - loss: 0.4943 - acc: 0.7943\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 479us/step - loss: 0.4793 - acc: 0.8005\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 484us/step - loss: 0.4473 - acc: 0.8155\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 484us/step - loss: 0.4356 - acc: 0.8204\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 504us/step - loss: 0.4387 - acc: 0.8204\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4401 - acc: 0.8329\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 509us/step - loss: 0.4315 - acc: 0.8242\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 484us/step - loss: 0.4354 - acc: 0.8229\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 499us/step - loss: 0.4114 - acc: 0.8279\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 484us/step - loss: 0.4230 - acc: 0.8142\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4248 - acc: 0.8342\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4085 - acc: 0.8367\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 494us/step - loss: 0.4181 - acc: 0.8267\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 484us/step - loss: 0.4182 - acc: 0.8329\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4114 - acc: 0.8329\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4219 - acc: 0.8242\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 564us/step - loss: 0.4050 - acc: 0.8441\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 539us/step - loss: 0.4088 - acc: 0.8317\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4102 - acc: 0.8304\n",
      "89/89 [==============================] - 1s 15ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 4s 4ms/step - loss: 0.6172 - acc: 0.6633\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 504us/step - loss: 0.4926 - acc: 0.7918\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 589us/step - loss: 0.4529 - acc: 0.8180\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 588us/step - loss: 0.4584 - acc: 0.8155\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 504us/step - loss: 0.4485 - acc: 0.8192\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 524us/step - loss: 0.4303 - acc: 0.8192\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 504us/step - loss: 0.4304 - acc: 0.8317\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 608us/step - loss: 0.4392 - acc: 0.8242\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 529us/step - loss: 0.4261 - acc: 0.8342\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 519us/step - loss: 0.4222 - acc: 0.8416\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 504us/step - loss: 0.4426 - acc: 0.8229\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 519us/step - loss: 0.4190 - acc: 0.8267\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 534us/step - loss: 0.4326 - acc: 0.8367\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 534us/step - loss: 0.4289 - acc: 0.8279\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 529us/step - loss: 0.4150 - acc: 0.8379\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 519us/step - loss: 0.4096 - acc: 0.8429\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 544us/step - loss: 0.4066 - acc: 0.8429\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 608us/step - loss: 0.3995 - acc: 0.8404\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 623us/step - loss: 0.4053 - acc: 0.8416\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 569us/step - loss: 0.4113 - acc: 0.8342\n",
      "89/89 [==============================] - 1s 16ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 4s 4ms/step - loss: 0.5928 - acc: 0.6646\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 514us/step - loss: 0.5080 - acc: 0.7893\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 519us/step - loss: 0.4952 - acc: 0.7943\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 514us/step - loss: 0.4719 - acc: 0.8092\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 519us/step - loss: 0.4604 - acc: 0.8130\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 519us/step - loss: 0.4686 - acc: 0.8180\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 524us/step - loss: 0.4532 - acc: 0.8254\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 529us/step - loss: 0.4499 - acc: 0.8167\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 613us/step - loss: 0.4618 - acc: 0.8130\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 519us/step - loss: 0.4535 - acc: 0.8304\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 514us/step - loss: 0.4447 - acc: 0.8304\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 539us/step - loss: 0.4486 - acc: 0.8229\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 529us/step - loss: 0.4398 - acc: 0.8217\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 519us/step - loss: 0.4497 - acc: 0.8242\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 524us/step - loss: 0.4459 - acc: 0.8379\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 529us/step - loss: 0.4369 - acc: 0.8279\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 509us/step - loss: 0.4387 - acc: 0.8354\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 519us/step - loss: 0.4342 - acc: 0.8292\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 514us/step - loss: 0.4306 - acc: 0.8217\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 1s 638us/step - loss: 0.4267 - acc: 0.8404\n",
      "89/89 [==============================] - 1s 16ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 4s 5ms/step - loss: 0.6106 - acc: 0.6771\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 623us/step - loss: 0.4888 - acc: 0.7968\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 593us/step - loss: 0.4595 - acc: 0.8167\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 1s 663us/step - loss: 0.4450 - acc: 0.8217\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 1s 838us/step - loss: 0.4516 - acc: 0.8229\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 618us/step - loss: 0.4453 - acc: 0.8292\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 1s 728us/step - loss: 0.4313 - acc: 0.8279\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 1s 638us/step - loss: 0.4461 - acc: 0.8292\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 593us/step - loss: 0.4194 - acc: 0.8279\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 494us/step - loss: 0.4273 - acc: 0.8292\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 549us/step - loss: 0.4234 - acc: 0.8229\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 584us/step - loss: 0.4268 - acc: 0.8242\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 569us/step - loss: 0.4157 - acc: 0.8279\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 569us/step - loss: 0.4226 - acc: 0.8354\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 509us/step - loss: 0.4111 - acc: 0.8354\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 529us/step - loss: 0.4112 - acc: 0.8354\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 589us/step - loss: 0.4156 - acc: 0.8342\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 599us/step - loss: 0.4223 - acc: 0.8329\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 579us/step - loss: 0.4172 - acc: 0.8379\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 574us/step - loss: 0.4101 - acc: 0.8392\n",
      "89/89 [==============================] - 1s 15ms/step\n",
      "Epoch 1/20\n",
      "803/803 [==============================] - 4s 5ms/step - loss: 0.6277 - acc: 0.6513\n",
      "Epoch 2/20\n",
      "803/803 [==============================] - 0s 608us/step - loss: 0.4924 - acc: 0.8045\n",
      "Epoch 3/20\n",
      "803/803 [==============================] - 0s 588us/step - loss: 0.4689 - acc: 0.8070\n",
      "Epoch 4/20\n",
      "803/803 [==============================] - 0s 513us/step - loss: 0.4589 - acc: 0.8107\n",
      "Epoch 5/20\n",
      "803/803 [==============================] - 0s 573us/step - loss: 0.4413 - acc: 0.8269\n",
      "Epoch 6/20\n",
      "803/803 [==============================] - 0s 618us/step - loss: 0.4428 - acc: 0.8194\n",
      "Epoch 7/20\n",
      "803/803 [==============================] - 0s 518us/step - loss: 0.4326 - acc: 0.8356\n",
      "Epoch 8/20\n",
      "803/803 [==============================] - 0s 583us/step - loss: 0.4571 - acc: 0.8281\n",
      "Epoch 9/20\n",
      "803/803 [==============================] - 0s 508us/step - loss: 0.4161 - acc: 0.8356\n",
      "Epoch 10/20\n",
      "803/803 [==============================] - 0s 508us/step - loss: 0.4347 - acc: 0.8232\n",
      "Epoch 11/20\n",
      "803/803 [==============================] - 0s 608us/step - loss: 0.4159 - acc: 0.8381\n",
      "Epoch 12/20\n",
      "803/803 [==============================] - 0s 618us/step - loss: 0.4215 - acc: 0.8406\n",
      "Epoch 13/20\n",
      "803/803 [==============================] - 1s 648us/step - loss: 0.4142 - acc: 0.8219\n",
      "Epoch 14/20\n",
      "803/803 [==============================] - 0s 593us/step - loss: 0.4192 - acc: 0.8294\n",
      "Epoch 15/20\n",
      "803/803 [==============================] - 0s 598us/step - loss: 0.4036 - acc: 0.8406\n",
      "Epoch 16/20\n",
      "803/803 [==============================] - 0s 513us/step - loss: 0.4165 - acc: 0.8356\n",
      "Epoch 17/20\n",
      "803/803 [==============================] - 0s 538us/step - loss: 0.4133 - acc: 0.8294\n",
      "Epoch 18/20\n",
      "803/803 [==============================] - 0s 538us/step - loss: 0.4123 - acc: 0.8406\n",
      "Epoch 19/20\n",
      "803/803 [==============================] - 0s 538us/step - loss: 0.4077 - acc: 0.8244\n",
      "Epoch 20/20\n",
      "803/803 [==============================] - 0s 528us/step - loss: 0.4037 - acc: 0.8356\n",
      "88/88 [==============================] - 1s 16ms/step\n",
      "Results: 83.16% (3.46%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "seed = 7\n",
    "np.random.seed(seed)\n",
    "def create_smaller_2():\n",
    "  model = models.Sequential()\n",
    "  adam = optimizers.Adam(lr = 0.002)\n",
    "  model.add(layers.Dense(60,activation = 'relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), \n",
    "                         input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dropout(0.40))\n",
    "  model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "  model.add(layers.Dropout(0.20))\n",
    "  model.add(layers.Dense(1,kernel_initializer = 'normal', activation = 'sigmoid'))\n",
    "  model.compile(optimizer = adam,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller_2, epochs=20, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "801/801 [==============================] - 6s 7ms/step - loss: 0.5824 - acc: 0.7004\n",
      "Epoch 2/20\n",
      "801/801 [==============================] - 1s 949us/step - loss: 0.4706 - acc: 0.7965\n",
      "Epoch 3/20\n",
      "801/801 [==============================] - 1s 799us/step - loss: 0.4779 - acc: 0.8040\n",
      "Epoch 4/20\n",
      "801/801 [==============================] - 1s 864us/step - loss: 0.4540 - acc: 0.8090\n",
      "Epoch 5/20\n",
      "801/801 [==============================] - 1s 849us/step - loss: 0.4284 - acc: 0.8265\n",
      "Epoch 6/20\n",
      "801/801 [==============================] - 1s 759us/step - loss: 0.4377 - acc: 0.8152\n",
      "Epoch 7/20\n",
      "801/801 [==============================] - 1s 794us/step - loss: 0.4273 - acc: 0.8190\n",
      "Epoch 8/20\n",
      "801/801 [==============================] - 1s 769us/step - loss: 0.4394 - acc: 0.8152\n",
      "Epoch 9/20\n",
      "801/801 [==============================] - 1s 749us/step - loss: 0.4367 - acc: 0.8215\n",
      "Epoch 10/20\n",
      "801/801 [==============================] - 1s 779us/step - loss: 0.4283 - acc: 0.8290\n",
      "Epoch 11/20\n",
      "801/801 [==============================] - 1s 779us/step - loss: 0.4261 - acc: 0.8252\n",
      "Epoch 12/20\n",
      "801/801 [==============================] - 1s 739us/step - loss: 0.4321 - acc: 0.8252\n",
      "Epoch 13/20\n",
      "801/801 [==============================] - 1s 854us/step - loss: 0.4294 - acc: 0.8290\n",
      "Epoch 14/20\n",
      "801/801 [==============================] - 1s 794us/step - loss: 0.4113 - acc: 0.8365\n",
      "Epoch 15/20\n",
      "801/801 [==============================] - 1s 849us/step - loss: 0.4202 - acc: 0.8352\n",
      "Epoch 16/20\n",
      "801/801 [==============================] - 1s 934us/step - loss: 0.4166 - acc: 0.8302\n",
      "Epoch 17/20\n",
      "801/801 [==============================] - 1s 1ms/step - loss: 0.4122 - acc: 0.8290A: 0s - loss: 0.360\n",
      "Epoch 18/20\n",
      "801/801 [==============================] - 1s 1ms/step - loss: 0.4124 - acc: 0.8290\n",
      "Epoch 19/20\n",
      "801/801 [==============================] - 1s 959us/step - loss: 0.4124 - acc: 0.8265\n",
      "Epoch 20/20\n",
      "801/801 [==============================] - 1s 1ms/step - loss: 0.4058 - acc: 0.8315\n",
      "90/90 [==============================] - 2s 28ms/step\n",
      "Epoch 1/20\n",
      "801/801 [==============================] - 6s 7ms/step - loss: 0.6072 - acc: 0.6667\n",
      "Epoch 2/20\n",
      "801/801 [==============================] - 1s 929us/step - loss: 0.4883 - acc: 0.7903\n",
      "Epoch 3/20\n",
      "801/801 [==============================] - 1s 989us/step - loss: 0.4826 - acc: 0.7953\n",
      "Epoch 4/20\n",
      "801/801 [==============================] - 1s 914us/step - loss: 0.4783 - acc: 0.8115\n",
      "Epoch 5/20\n",
      "801/801 [==============================] - 1s 1ms/step - loss: 0.4548 - acc: 0.8077\n",
      "Epoch 6/20\n",
      "801/801 [==============================] - 1s 954us/step - loss: 0.4459 - acc: 0.8102\n",
      "Epoch 7/20\n",
      "801/801 [==============================] - 1s 769us/step - loss: 0.4472 - acc: 0.8202\n",
      "Epoch 8/20\n",
      "801/801 [==============================] - 1s 779us/step - loss: 0.4396 - acc: 0.8140\n",
      "Epoch 9/20\n",
      "801/801 [==============================] - 1s 915us/step - loss: 0.4461 - acc: 0.8227\n",
      "Epoch 10/20\n",
      "801/801 [==============================] - 1s 844us/step - loss: 0.4348 - acc: 0.8277\n",
      "Epoch 11/20\n",
      "801/801 [==============================] - 1s 844us/step - loss: 0.4449 - acc: 0.8190\n",
      "Epoch 12/20\n",
      "801/801 [==============================] - 1s 869us/step - loss: 0.4256 - acc: 0.8277\n",
      "Epoch 13/20\n",
      "801/801 [==============================] - 1s 899us/step - loss: 0.4278 - acc: 0.8240\n",
      "Epoch 14/20\n",
      "801/801 [==============================] - 1s 1ms/step - loss: 0.4357 - acc: 0.8290\n",
      "Epoch 15/20\n",
      "801/801 [==============================] - 1s 979us/step - loss: 0.4317 - acc: 0.8340\n",
      "Epoch 16/20\n",
      "801/801 [==============================] - 1s 829us/step - loss: 0.4196 - acc: 0.8327\n",
      "Epoch 17/20\n",
      "801/801 [==============================] - 1s 799us/step - loss: 0.4244 - acc: 0.8327\n",
      "Epoch 18/20\n",
      "801/801 [==============================] - 1s 989us/step - loss: 0.4316 - acc: 0.8252\n",
      "Epoch 19/20\n",
      "801/801 [==============================] - 1s 884us/step - loss: 0.4137 - acc: 0.8365\n",
      "Epoch 20/20\n",
      "801/801 [==============================] - 1s 784us/step - loss: 0.4126 - acc: 0.8315\n",
      "90/90 [==============================] - 2s 24ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 8ms/step - loss: 0.5872 - acc: 0.6920\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 1s 973us/step - loss: 0.5061 - acc: 0.7955\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 1s 898us/step - loss: 0.4814 - acc: 0.7943\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 1s 953us/step - loss: 0.4675 - acc: 0.8105\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4678 - acc: 0.8192\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 1s 958us/step - loss: 0.4463 - acc: 0.8192\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4507 - acc: 0.8229\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 1s 959us/step - loss: 0.4578 - acc: 0.8180\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 1s 913us/step - loss: 0.4475 - acc: 0.8117\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 1s 838us/step - loss: 0.4316 - acc: 0.8192\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 1s 768us/step - loss: 0.4329 - acc: 0.8292\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 1s 813us/step - loss: 0.4483 - acc: 0.8155\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 1s 788us/step - loss: 0.4306 - acc: 0.8204\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 1s 779us/step - loss: 0.4224 - acc: 0.8142\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 1s 803us/step - loss: 0.4300 - acc: 0.8254\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 1s 813us/step - loss: 0.4354 - acc: 0.8242\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 1s 823us/step - loss: 0.4212 - acc: 0.8229\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 1s 868us/step - loss: 0.4230 - acc: 0.8317\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 1s 943us/step - loss: 0.4181 - acc: 0.8379\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4252 - acc: 0.8354\n",
      "89/89 [==============================] - 2s 28ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 8ms/step - loss: 0.5928 - acc: 0.6908\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 1s 808us/step - loss: 0.4756 - acc: 0.8067\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 1s 863us/step - loss: 0.4671 - acc: 0.8042\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 1s 893us/step - loss: 0.4623 - acc: 0.8167\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4385 - acc: 0.8279\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4436 - acc: 0.8279\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4447 - acc: 0.8342\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4368 - acc: 0.8317\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4438 - acc: 0.8304\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4435 - acc: 0.8229\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4201 - acc: 0.8317A: 0s - loss: 0.3\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4211 - acc: 0.8254\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4449 - acc: 0.8267\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 1s 957us/step - loss: 0.4208 - acc: 0.8342\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 1s 753us/step - loss: 0.4104 - acc: 0.8317\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 1s 745us/step - loss: 0.4042 - acc: 0.8392\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 1s 745us/step - loss: 0.4153 - acc: 0.8454\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 1s 844us/step - loss: 0.4023 - acc: 0.8454\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 1s 879us/step - loss: 0.4307 - acc: 0.8317\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 1s 765us/step - loss: 0.4280 - acc: 0.8304\n",
      "89/89 [==============================] - 2s 25ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 7s 8ms/step - loss: 0.5828 - acc: 0.6920\n",
      "Epoch 2/20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "802/802 [==============================] - 1s 768us/step - loss: 0.4894 - acc: 0.8080\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 1s 653us/step - loss: 0.4654 - acc: 0.8217\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 1s 748us/step - loss: 0.4447 - acc: 0.8217\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 604us/step - loss: 0.4362 - acc: 0.8217\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 608us/step - loss: 0.4399 - acc: 0.8379\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 609us/step - loss: 0.4254 - acc: 0.8267\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 1s 798us/step - loss: 0.4278 - acc: 0.8292\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 1s 688us/step - loss: 0.4228 - acc: 0.8354\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 1s 833us/step - loss: 0.4219 - acc: 0.8229\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 1s 878us/step - loss: 0.4112 - acc: 0.8342\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 1s 868us/step - loss: 0.4264 - acc: 0.8279\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 1s 913us/step - loss: 0.4196 - acc: 0.8354\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4104 - acc: 0.8379\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4027 - acc: 0.8317\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 1s 943us/step - loss: 0.4042 - acc: 0.8416\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 1s 923us/step - loss: 0.4081 - acc: 0.8441\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 1s 883us/step - loss: 0.3986 - acc: 0.8441\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 1s 963us/step - loss: 0.4005 - acc: 0.8379\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 1s 958us/step - loss: 0.4057 - acc: 0.8367\n",
      "89/89 [==============================] - 2s 25ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 7ms/step - loss: 0.5993 - acc: 0.6796\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 1s 803us/step - loss: 0.5001 - acc: 0.7930\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 1s 793us/step - loss: 0.4842 - acc: 0.8005\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 1s 803us/step - loss: 0.4563 - acc: 0.8229\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 1s 808us/step - loss: 0.4454 - acc: 0.8155\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 1s 833us/step - loss: 0.4394 - acc: 0.8242\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 1s 828us/step - loss: 0.4421 - acc: 0.8180\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 1s 898us/step - loss: 0.4391 - acc: 0.8204\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 1s 818us/step - loss: 0.4484 - acc: 0.8155\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 1s 943us/step - loss: 0.4184 - acc: 0.8379\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 1s 913us/step - loss: 0.4235 - acc: 0.8267\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 1s 903us/step - loss: 0.4409 - acc: 0.8367\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 1s 978us/step - loss: 0.4123 - acc: 0.8292\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 1s 958us/step - loss: 0.4174 - acc: 0.8342\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 1s 928us/step - loss: 0.4121 - acc: 0.8292\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 1s 828us/step - loss: 0.4148 - acc: 0.8429\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 1s 838us/step - loss: 0.4256 - acc: 0.8267\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 1s 803us/step - loss: 0.4106 - acc: 0.8329\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 1s 823us/step - loss: 0.4043 - acc: 0.8279\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 1s 818us/step - loss: 0.4254 - acc: 0.8254\n",
      "89/89 [==============================] - 2s 25ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 7ms/step - loss: 0.6041 - acc: 0.6833\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 1s 823us/step - loss: 0.4995 - acc: 0.7918\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 1s 814us/step - loss: 0.4600 - acc: 0.8204\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 1s 848us/step - loss: 0.4670 - acc: 0.8092\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 1s 813us/step - loss: 0.4560 - acc: 0.8279\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 1s 833us/step - loss: 0.4341 - acc: 0.8229\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 1s 923us/step - loss: 0.4271 - acc: 0.8416\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 1s 997us/step - loss: 0.4435 - acc: 0.8292\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 1s 928us/step - loss: 0.4242 - acc: 0.8304\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 1s 913us/step - loss: 0.4321 - acc: 0.8279\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 1s 958us/step - loss: 0.4483 - acc: 0.8180\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 1s 913us/step - loss: 0.4353 - acc: 0.8229\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4323 - acc: 0.8329\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 1s 958us/step - loss: 0.4299 - acc: 0.8354\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 1s 958us/step - loss: 0.4140 - acc: 0.8292\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 1s 818us/step - loss: 0.4104 - acc: 0.8392\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 1s 788us/step - loss: 0.4197 - acc: 0.8342\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 1s 788us/step - loss: 0.4029 - acc: 0.8429\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 1s 963us/step - loss: 0.4103 - acc: 0.8379\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 1s 923us/step - loss: 0.4117 - acc: 0.8342\n",
      "89/89 [==============================] - 3s 30ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 8ms/step - loss: 0.5788 - acc: 0.6870\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 1s 973us/step - loss: 0.5005 - acc: 0.8055\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4776 - acc: 0.8042\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 1s 858us/step - loss: 0.4662 - acc: 0.8117\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 1s 913us/step - loss: 0.4671 - acc: 0.8092\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 1s 913us/step - loss: 0.4671 - acc: 0.8080\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 1s 973us/step - loss: 0.4457 - acc: 0.8267\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 1s 873us/step - loss: 0.4466 - acc: 0.8130\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 1s 863us/step - loss: 0.4571 - acc: 0.8142\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 1s 963us/step - loss: 0.4360 - acc: 0.8242\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 1s 928us/step - loss: 0.4440 - acc: 0.8180\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4393 - acc: 0.8242\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4454 - acc: 0.8279\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4572 - acc: 0.8242\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4338 - acc: 0.8367\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 1s 934us/step - loss: 0.4456 - acc: 0.8155\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 1s 1ms/step - loss: 0.4305 - acc: 0.8317\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 1s 908us/step - loss: 0.4246 - acc: 0.8267\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 1s 629us/step - loss: 0.4353 - acc: 0.8342\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 602us/step - loss: 0.4243 - acc: 0.8441\n",
      "89/89 [==============================] - 2s 26ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 7ms/step - loss: 0.5963 - acc: 0.6933\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 615us/step - loss: 0.4885 - acc: 0.7955\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 1s 630us/step - loss: 0.4716 - acc: 0.8155\n",
      "Epoch 4/20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "802/802 [==============================] - 0s 605us/step - loss: 0.4446 - acc: 0.8279\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 1s 730us/step - loss: 0.4643 - acc: 0.8180\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 1s 720us/step - loss: 0.4557 - acc: 0.8292\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 607us/step - loss: 0.4355 - acc: 0.8180\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 594us/step - loss: 0.4506 - acc: 0.8204\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 590us/step - loss: 0.4219 - acc: 0.8317\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 622us/step - loss: 0.4382 - acc: 0.8192\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 591us/step - loss: 0.4320 - acc: 0.8279\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 623us/step - loss: 0.4332 - acc: 0.8354\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 591us/step - loss: 0.4181 - acc: 0.8292\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 591us/step - loss: 0.4291 - acc: 0.8329\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 602us/step - loss: 0.4204 - acc: 0.8354\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 591us/step - loss: 0.4031 - acc: 0.8317\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 620us/step - loss: 0.4154 - acc: 0.8329\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 1s 626us/step - loss: 0.4310 - acc: 0.8354\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 596us/step - loss: 0.4229 - acc: 0.8354\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 614us/step - loss: 0.4225 - acc: 0.8317\n",
      "89/89 [==============================] - 2s 26ms/step\n",
      "Epoch 1/20\n",
      "803/803 [==============================] - 7s 9ms/step - loss: 0.6101 - acc: 0.6787\n",
      "Epoch 2/20\n",
      "803/803 [==============================] - 1s 1ms/step - loss: 0.4936 - acc: 0.8032\n",
      "Epoch 3/20\n",
      "803/803 [==============================] - 1s 833us/step - loss: 0.4695 - acc: 0.8070\n",
      "Epoch 4/20\n",
      "803/803 [==============================] - 1s 932us/step - loss: 0.4676 - acc: 0.8132\n",
      "Epoch 5/20\n",
      "803/803 [==============================] - 1s 1ms/step - loss: 0.4392 - acc: 0.8232\n",
      "Epoch 6/20\n",
      "803/803 [==============================] - 1s 921us/step - loss: 0.4502 - acc: 0.8182\n",
      "Epoch 7/20\n",
      "803/803 [==============================] - 1s 913us/step - loss: 0.4387 - acc: 0.8257\n",
      "Epoch 8/20\n",
      "803/803 [==============================] - 1s 832us/step - loss: 0.4554 - acc: 0.8169\n",
      "Epoch 9/20\n",
      "803/803 [==============================] - 1s 770us/step - loss: 0.4316 - acc: 0.8369\n",
      "Epoch 10/20\n",
      "803/803 [==============================] - 1s 852us/step - loss: 0.4423 - acc: 0.8306\n",
      "Epoch 11/20\n",
      "803/803 [==============================] - 1s 890us/step - loss: 0.4272 - acc: 0.8319\n",
      "Epoch 12/20\n",
      "803/803 [==============================] - 1s 989us/step - loss: 0.4366 - acc: 0.8207\n",
      "Epoch 13/20\n",
      "803/803 [==============================] - 1s 834us/step - loss: 0.4273 - acc: 0.8232\n",
      "Epoch 14/20\n",
      "803/803 [==============================] - 1s 922us/step - loss: 0.4179 - acc: 0.8294\n",
      "Epoch 15/20\n",
      "803/803 [==============================] - 1s 868us/step - loss: 0.4189 - acc: 0.8406\n",
      "Epoch 16/20\n",
      "803/803 [==============================] - 1s 1ms/step - loss: 0.4297 - acc: 0.8331\n",
      "Epoch 17/20\n",
      "803/803 [==============================] - 1s 978us/step - loss: 0.4295 - acc: 0.8157\n",
      "Epoch 18/20\n",
      "803/803 [==============================] - 1s 929us/step - loss: 0.4130 - acc: 0.8394\n",
      "Epoch 19/20\n",
      "803/803 [==============================] - 1s 906us/step - loss: 0.4182 - acc: 0.8257\n",
      "Epoch 20/20\n",
      "803/803 [==============================] - 1s 912us/step - loss: 0.4032 - acc: 0.8418\n",
      "88/88 [==============================] - 2s 28ms/step\n",
      "Results: 83.39% (3.13%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "seed = 7\n",
    "np.random.seed(seed)\n",
    "def create_smaller_2():\n",
    "  model = models.Sequential()\n",
    "  adam = optimizers.Adam(lr = 0.003)\n",
    "  model.add(layers.Dense(60,activation = 'relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), \n",
    "                         input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dropout(0.50))\n",
    "  model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "  model.add(layers.Dropout(0.20))\n",
    "  model.add(layers.Dense(1,kernel_initializer = 'normal', activation = 'sigmoid'))\n",
    "  model.compile(optimizer = adam,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller_2, epochs=20, batch_size=5, verbose=0)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "801/801 [==============================] - 4s 5ms/step - loss: 0.5809 - acc: 0.7029\n",
      "Epoch 2/20\n",
      "801/801 [==============================] - ETA: 0s - loss: 0.4790 - acc: 0.786 - 0s 300us/step - loss: 0.4774 - acc: 0.7878\n",
      "Epoch 3/20\n",
      "801/801 [==============================] - 0s 290us/step - loss: 0.4615 - acc: 0.8127\n",
      "Epoch 4/20\n",
      "801/801 [==============================] - 0s 275us/step - loss: 0.4657 - acc: 0.8065 0s - loss: 0.3828 - acc: 0.\n",
      "Epoch 5/20\n",
      "801/801 [==============================] - 0s 285us/step - loss: 0.4551 - acc: 0.8190\n",
      "Epoch 6/20\n",
      "801/801 [==============================] - 0s 285us/step - loss: 0.4410 - acc: 0.8240\n",
      "Epoch 7/20\n",
      "801/801 [==============================] - 0s 295us/step - loss: 0.4203 - acc: 0.8290\n",
      "Epoch 8/20\n",
      "801/801 [==============================] - 0s 295us/step - loss: 0.4347 - acc: 0.8215\n",
      "Epoch 9/20\n",
      "801/801 [==============================] - 0s 335us/step - loss: 0.4345 - acc: 0.8152\n",
      "Epoch 10/20\n",
      "801/801 [==============================] - 0s 330us/step - loss: 0.4143 - acc: 0.8215\n",
      "Epoch 11/20\n",
      "801/801 [==============================] - 0s 315us/step - loss: 0.4227 - acc: 0.8277\n",
      "Epoch 12/20\n",
      "801/801 [==============================] - 0s 300us/step - loss: 0.4185 - acc: 0.8277\n",
      "Epoch 13/20\n",
      "801/801 [==============================] - 0s 335us/step - loss: 0.4279 - acc: 0.8227\n",
      "Epoch 14/20\n",
      "801/801 [==============================] - 0s 315us/step - loss: 0.4077 - acc: 0.8240\n",
      "Epoch 15/20\n",
      "801/801 [==============================] - 0s 305us/step - loss: 0.4082 - acc: 0.8340\n",
      "Epoch 16/20\n",
      "801/801 [==============================] - 0s 335us/step - loss: 0.4130 - acc: 0.8365\n",
      "Epoch 17/20\n",
      "801/801 [==============================] - 0s 320us/step - loss: 0.4074 - acc: 0.8439\n",
      "Epoch 18/20\n",
      "801/801 [==============================] - 0s 300us/step - loss: 0.4044 - acc: 0.8402\n",
      "Epoch 19/20\n",
      "801/801 [==============================] - 0s 325us/step - loss: 0.4091 - acc: 0.8352\n",
      "Epoch 20/20\n",
      "801/801 [==============================] - 0s 305us/step - loss: 0.4033 - acc: 0.8489\n",
      "90/90 [==============================] - 2s 19ms/step\n",
      "Epoch 1/20\n",
      "801/801 [==============================] - 4s 5ms/step - loss: 0.5985 - acc: 0.6667\n",
      "Epoch 2/20\n",
      "801/801 [==============================] - 0s 325us/step - loss: 0.5005 - acc: 0.7890\n",
      "Epoch 3/20\n",
      "801/801 [==============================] - 0s 449us/step - loss: 0.4758 - acc: 0.7990\n",
      "Epoch 4/20\n",
      "801/801 [==============================] - 0s 434us/step - loss: 0.4654 - acc: 0.8115\n",
      "Epoch 5/20\n",
      "801/801 [==============================] - 0s 429us/step - loss: 0.4507 - acc: 0.8165\n",
      "Epoch 6/20\n",
      "801/801 [==============================] - 0s 404us/step - loss: 0.4342 - acc: 0.8215\n",
      "Epoch 7/20\n",
      "801/801 [==============================] - 0s 464us/step - loss: 0.4349 - acc: 0.8115\n",
      "Epoch 8/20\n",
      "801/801 [==============================] - 0s 454us/step - loss: 0.4456 - acc: 0.8277\n",
      "Epoch 9/20\n",
      "801/801 [==============================] - 0s 390us/step - loss: 0.4449 - acc: 0.8315\n",
      "Epoch 10/20\n",
      "801/801 [==============================] - 0s 430us/step - loss: 0.4149 - acc: 0.8377\n",
      "Epoch 11/20\n",
      "801/801 [==============================] - 0s 469us/step - loss: 0.4344 - acc: 0.8265\n",
      "Epoch 12/20\n",
      "801/801 [==============================] - 0s 355us/step - loss: 0.4182 - acc: 0.8302\n",
      "Epoch 13/20\n",
      "801/801 [==============================] - 0s 454us/step - loss: 0.4189 - acc: 0.8240\n",
      "Epoch 14/20\n",
      "801/801 [==============================] - 0s 529us/step - loss: 0.4247 - acc: 0.8240\n",
      "Epoch 15/20\n",
      "801/801 [==============================] - 0s 579us/step - loss: 0.4210 - acc: 0.8390\n",
      "Epoch 16/20\n",
      "801/801 [==============================] - 0s 444us/step - loss: 0.4301 - acc: 0.8227\n",
      "Epoch 17/20\n",
      "801/801 [==============================] - 0s 390us/step - loss: 0.4170 - acc: 0.8340\n",
      "Epoch 18/20\n",
      "801/801 [==============================] - 0s 325us/step - loss: 0.4205 - acc: 0.8265\n",
      "Epoch 19/20\n",
      "801/801 [==============================] - 0s 355us/step - loss: 0.4348 - acc: 0.8290\n",
      "Epoch 20/20\n",
      "801/801 [==============================] - 0s 355us/step - loss: 0.4063 - acc: 0.8340\n",
      "90/90 [==============================] - 2s 19ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 4s 5ms/step - loss: 0.5888 - acc: 0.6945\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 329us/step - loss: 0.5049 - acc: 0.8080\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 314us/step - loss: 0.4668 - acc: 0.8055\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4493 - acc: 0.8142\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4610 - acc: 0.8055\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 314us/step - loss: 0.4391 - acc: 0.8254\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4402 - acc: 0.8267\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 324us/step - loss: 0.4360 - acc: 0.8229\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 314us/step - loss: 0.4421 - acc: 0.8105\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 324us/step - loss: 0.4135 - acc: 0.8229\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 334us/step - loss: 0.4216 - acc: 0.8304\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4311 - acc: 0.8267\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 309us/step - loss: 0.4301 - acc: 0.8180\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 314us/step - loss: 0.4247 - acc: 0.8242\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 324us/step - loss: 0.4121 - acc: 0.8317\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4240 - acc: 0.8242\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 354us/step - loss: 0.4145 - acc: 0.8192\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 349us/step - loss: 0.4118 - acc: 0.8267\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 344us/step - loss: 0.4304 - acc: 0.8242\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 329us/step - loss: 0.4155 - acc: 0.8317\n",
      "89/89 [==============================] - 2s 18ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 4s 5ms/step - loss: 0.6029 - acc: 0.6958\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 309us/step - loss: 0.4725 - acc: 0.8105\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4759 - acc: 0.8067\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 349us/step - loss: 0.4510 - acc: 0.8192\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 349us/step - loss: 0.4427 - acc: 0.8204\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 334us/step - loss: 0.4483 - acc: 0.8342\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 334us/step - loss: 0.4383 - acc: 0.8329\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 329us/step - loss: 0.4402 - acc: 0.8155\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4425 - acc: 0.8342\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4402 - acc: 0.8204\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 429us/step - loss: 0.4220 - acc: 0.8317\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 329us/step - loss: 0.4298 - acc: 0.8354\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 404us/step - loss: 0.4252 - acc: 0.8342\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 429us/step - loss: 0.4182 - acc: 0.8304\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 524us/step - loss: 0.4176 - acc: 0.8304\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 479us/step - loss: 0.4094 - acc: 0.8354\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 359us/step - loss: 0.4038 - acc: 0.8379\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 294us/step - loss: 0.4080 - acc: 0.8379\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 304us/step - loss: 0.4185 - acc: 0.8392\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 304us/step - loss: 0.4242 - acc: 0.8267\n",
      "89/89 [==============================] - 2s 22ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 4s 6ms/step - loss: 0.5635 - acc: 0.6995\n",
      "Epoch 2/20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "802/802 [==============================] - 0s 239us/step - loss: 0.4756 - acc: 0.8130\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 259us/step - loss: 0.4568 - acc: 0.8229\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 264us/step - loss: 0.4575 - acc: 0.8267\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 254us/step - loss: 0.4372 - acc: 0.8317\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 249us/step - loss: 0.4146 - acc: 0.8392\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 279us/step - loss: 0.4270 - acc: 0.8329\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 249us/step - loss: 0.4200 - acc: 0.8354\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 274us/step - loss: 0.4166 - acc: 0.8317\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 254us/step - loss: 0.4164 - acc: 0.8329\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 249us/step - loss: 0.4076 - acc: 0.8429\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 249us/step - loss: 0.4179 - acc: 0.8367\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 244us/step - loss: 0.4107 - acc: 0.8342\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 244us/step - loss: 0.3949 - acc: 0.8416\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 249us/step - loss: 0.4096 - acc: 0.8242\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 244us/step - loss: 0.4050 - acc: 0.8504\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 244us/step - loss: 0.4139 - acc: 0.8317\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 249us/step - loss: 0.4031 - acc: 0.8342\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 264us/step - loss: 0.4029 - acc: 0.8404\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 274us/step - loss: 0.3974 - acc: 0.8367\n",
      "89/89 [==============================] - 2s 22ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 5s 6ms/step - loss: 0.5923 - acc: 0.6621\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 404us/step - loss: 0.4885 - acc: 0.8080\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 324us/step - loss: 0.4854 - acc: 0.8055\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 369us/step - loss: 0.4510 - acc: 0.8167\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4557 - acc: 0.8155\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 404us/step - loss: 0.4636 - acc: 0.8217\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4349 - acc: 0.8342\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 354us/step - loss: 0.4412 - acc: 0.8217\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 344us/step - loss: 0.4464 - acc: 0.8180\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 329us/step - loss: 0.4287 - acc: 0.8342\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 329us/step - loss: 0.4471 - acc: 0.8254\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4343 - acc: 0.8329\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 329us/step - loss: 0.4248 - acc: 0.8304\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 314us/step - loss: 0.4327 - acc: 0.8379\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4270 - acc: 0.8367\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4409 - acc: 0.8254\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 319us/step - loss: 0.4271 - acc: 0.8242\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 314us/step - loss: 0.4158 - acc: 0.8292\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 324us/step - loss: 0.4305 - acc: 0.8279\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 314us/step - loss: 0.4128 - acc: 0.8354\n",
      "89/89 [==============================] - 2s 22ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 4s 5ms/step - loss: 0.5919 - acc: 0.6958\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 314us/step - loss: 0.4815 - acc: 0.8055\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 324us/step - loss: 0.4577 - acc: 0.8142\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 329us/step - loss: 0.4516 - acc: 0.8192\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 314us/step - loss: 0.4497 - acc: 0.8292\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 339us/step - loss: 0.4306 - acc: 0.8180\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4281 - acc: 0.8317\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 374us/step - loss: 0.4267 - acc: 0.8279\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 434us/step - loss: 0.4198 - acc: 0.8267\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 369us/step - loss: 0.4216 - acc: 0.8304\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 424us/step - loss: 0.4259 - acc: 0.8292\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 359us/step - loss: 0.4207 - acc: 0.8354\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4135 - acc: 0.8429\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 414us/step - loss: 0.4272 - acc: 0.8267\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 404us/step - loss: 0.4201 - acc: 0.8392\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 409us/step - loss: 0.3995 - acc: 0.8441\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 349us/step - loss: 0.4256 - acc: 0.8342\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 404us/step - loss: 0.3933 - acc: 0.8454\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 354us/step - loss: 0.3992 - acc: 0.8304\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 339us/step - loss: 0.4172 - acc: 0.8254\n",
      "89/89 [==============================] - 2s 20ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 4s 5ms/step - loss: 0.5784 - acc: 0.7020\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 334us/step - loss: 0.5042 - acc: 0.7943\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 329us/step - loss: 0.4930 - acc: 0.8030\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 324us/step - loss: 0.4640 - acc: 0.8180\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 344us/step - loss: 0.4725 - acc: 0.8180\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 344us/step - loss: 0.4641 - acc: 0.8180\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 339us/step - loss: 0.4578 - acc: 0.8192\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 349us/step - loss: 0.4337 - acc: 0.8254\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 339us/step - loss: 0.4581 - acc: 0.8267\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 344us/step - loss: 0.4569 - acc: 0.8167\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 354us/step - loss: 0.4572 - acc: 0.8292\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 344us/step - loss: 0.4578 - acc: 0.8217\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 354us/step - loss: 0.4517 - acc: 0.8192\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 334us/step - loss: 0.4557 - acc: 0.8229\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 344us/step - loss: 0.4429 - acc: 0.8279\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 339us/step - loss: 0.4343 - acc: 0.8242\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 344us/step - loss: 0.4343 - acc: 0.8342\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 369us/step - loss: 0.4277 - acc: 0.8217\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 344us/step - loss: 0.4341 - acc: 0.8317\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 349us/step - loss: 0.4312 - acc: 0.8267\n",
      "89/89 [==============================] - 2s 20ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 4s 5ms/step - loss: 0.6043 - acc: 0.6322\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 339us/step - loss: 0.5125 - acc: 0.7968\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 434us/step - loss: 0.4788 - acc: 0.8254\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 439us/step - loss: 0.4583 - acc: 0.8167\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 369us/step - loss: 0.4424 - acc: 0.8254\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 339us/step - loss: 0.4487 - acc: 0.8204\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 339us/step - loss: 0.4268 - acc: 0.8242\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 334us/step - loss: 0.4359 - acc: 0.8267\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 324us/step - loss: 0.4341 - acc: 0.8367\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 374us/step - loss: 0.4255 - acc: 0.8317\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4319 - acc: 0.8229\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 409us/step - loss: 0.4285 - acc: 0.8292\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 434us/step - loss: 0.4350 - acc: 0.8242\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 349us/step - loss: 0.4241 - acc: 0.8329\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 464us/step - loss: 0.4194 - acc: 0.8429 0s - loss: 0.3504 - acc: \n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 424us/step - loss: 0.4165 - acc: 0.8292\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 429us/step - loss: 0.4145 - acc: 0.8404\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 329us/step - loss: 0.4305 - acc: 0.8292\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 364us/step - loss: 0.4045 - acc: 0.8441\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 394us/step - loss: 0.4073 - acc: 0.8329\n",
      "89/89 [==============================] - 2s 20ms/step\n",
      "Epoch 1/20\n",
      "803/803 [==============================] - 5s 6ms/step - loss: 0.6003 - acc: 0.6924\n",
      "Epoch 2/20\n",
      "803/803 [==============================] - 0s 488us/step - loss: 0.4804 - acc: 0.8057\n",
      "Epoch 3/20\n",
      "803/803 [==============================] - 0s 359us/step - loss: 0.4578 - acc: 0.8207\n",
      "Epoch 4/20\n",
      "803/803 [==============================] - 0s 354us/step - loss: 0.4469 - acc: 0.8219\n",
      "Epoch 5/20\n",
      "803/803 [==============================] - 0s 364us/step - loss: 0.4356 - acc: 0.8219\n",
      "Epoch 6/20\n",
      "803/803 [==============================] - 0s 339us/step - loss: 0.4443 - acc: 0.8207\n",
      "Epoch 7/20\n",
      "803/803 [==============================] - 0s 433us/step - loss: 0.4371 - acc: 0.8294\n",
      "Epoch 8/20\n",
      "803/803 [==============================] - 0s 488us/step - loss: 0.4403 - acc: 0.8294\n",
      "Epoch 9/20\n",
      "803/803 [==============================] - 0s 453us/step - loss: 0.4354 - acc: 0.8306\n",
      "Epoch 10/20\n",
      "803/803 [==============================] - 0s 389us/step - loss: 0.4295 - acc: 0.8269\n",
      "Epoch 11/20\n",
      "803/803 [==============================] - 0s 418us/step - loss: 0.4194 - acc: 0.8344\n",
      "Epoch 12/20\n",
      "803/803 [==============================] - 0s 403us/step - loss: 0.4157 - acc: 0.8344\n",
      "Epoch 13/20\n",
      "803/803 [==============================] - 0s 389us/step - loss: 0.4215 - acc: 0.8394\n",
      "Epoch 14/20\n",
      "803/803 [==============================] - 0s 364us/step - loss: 0.4209 - acc: 0.8418\n",
      "Epoch 15/20\n",
      "803/803 [==============================] - 0s 349us/step - loss: 0.4091 - acc: 0.8281\n",
      "Epoch 16/20\n",
      "803/803 [==============================] - 0s 433us/step - loss: 0.4175 - acc: 0.8331\n",
      "Epoch 17/20\n",
      "803/803 [==============================] - 0s 423us/step - loss: 0.4078 - acc: 0.8369\n",
      "Epoch 18/20\n",
      "803/803 [==============================] - 0s 354us/step - loss: 0.4179 - acc: 0.8306\n",
      "Epoch 19/20\n",
      "803/803 [==============================] - 0s 349us/step - loss: 0.4254 - acc: 0.8381\n",
      "Epoch 20/20\n",
      "803/803 [==============================] - 0s 423us/step - loss: 0.4178 - acc: 0.8431\n",
      "88/88 [==============================] - 2s 20ms/step\n",
      "Results: 82.83% (3.38%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "seed = 7\n",
    "np.random.seed(seed)\n",
    "def create_smaller_2():\n",
    "  model = models.Sequential()\n",
    "  adam = optimizers.Adam(lr = 0.005)\n",
    "  model.add(layers.Dense(60,activation = 'relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), \n",
    "                         input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dropout(0.35))\n",
    "  model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "  model.add(layers.Dropout(0.20))\n",
    "  model.add(layers.Dense(1,kernel_initializer = 'normal', activation = 'sigmoid'))\n",
    "  model.compile(optimizer = adam,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller_2, epochs=20, batch_size=10, verbose=1)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "801/801 [==============================] - 4s 5ms/step - loss: 0.5805 - acc: 0.7029\n",
      "Epoch 2/15\n",
      "801/801 [==============================] - 0s 315us/step - loss: 0.4778 - acc: 0.7903\n",
      "Epoch 3/15\n",
      "801/801 [==============================] - 0s 310us/step - loss: 0.4597 - acc: 0.8115\n",
      "Epoch 4/15\n",
      "801/801 [==============================] - 0s 315us/step - loss: 0.4638 - acc: 0.8040\n",
      "Epoch 5/15\n",
      "801/801 [==============================] - 0s 305us/step - loss: 0.4564 - acc: 0.8177\n",
      "Epoch 6/15\n",
      "801/801 [==============================] - 0s 315us/step - loss: 0.4394 - acc: 0.8277\n",
      "Epoch 7/15\n",
      "801/801 [==============================] - 0s 310us/step - loss: 0.4184 - acc: 0.8327\n",
      "Epoch 8/15\n",
      "801/801 [==============================] - 0s 360us/step - loss: 0.4353 - acc: 0.8215\n",
      "Epoch 9/15\n",
      "801/801 [==============================] - 0s 390us/step - loss: 0.4376 - acc: 0.8140 0s - loss: 0.4349 - acc: 0.8\n",
      "Epoch 10/15\n",
      "801/801 [==============================] - 0s 305us/step - loss: 0.4095 - acc: 0.8265\n",
      "Epoch 11/15\n",
      "801/801 [==============================] - 0s 325us/step - loss: 0.4304 - acc: 0.8277\n",
      "Epoch 12/15\n",
      "801/801 [==============================] - 0s 395us/step - loss: 0.4220 - acc: 0.8240\n",
      "Epoch 13/15\n",
      "801/801 [==============================] - 0s 345us/step - loss: 0.4259 - acc: 0.8240\n",
      "Epoch 14/15\n",
      "801/801 [==============================] - 0s 360us/step - loss: 0.4089 - acc: 0.8190\n",
      "Epoch 15/15\n",
      "801/801 [==============================] - 0s 395us/step - loss: 0.4103 - acc: 0.8265\n",
      "90/90 [==============================] - 2s 19ms/step\n",
      "Epoch 1/15\n",
      "801/801 [==============================] - 4s 5ms/step - loss: 0.5852 - acc: 0.6767\n",
      "Epoch 2/15\n",
      "801/801 [==============================] - 0s 390us/step - loss: 0.4917 - acc: 0.7990\n",
      "Epoch 3/15\n",
      "801/801 [==============================] - 0s 340us/step - loss: 0.4681 - acc: 0.8065\n",
      "Epoch 4/15\n",
      "801/801 [==============================] - 0s 429us/step - loss: 0.4533 - acc: 0.8152\n",
      "Epoch 5/15\n",
      "801/801 [==============================] - 0s 430us/step - loss: 0.4645 - acc: 0.8065\n",
      "Epoch 6/15\n",
      "801/801 [==============================] - 0s 434us/step - loss: 0.4374 - acc: 0.8140\n",
      "Epoch 7/15\n",
      "801/801 [==============================] - 0s 429us/step - loss: 0.4427 - acc: 0.8215\n",
      "Epoch 8/15\n",
      "801/801 [==============================] - 0s 439us/step - loss: 0.4325 - acc: 0.8302\n",
      "Epoch 9/15\n",
      "801/801 [==============================] - 0s 439us/step - loss: 0.4386 - acc: 0.8177\n",
      "Epoch 10/15\n",
      "801/801 [==============================] - 0s 454us/step - loss: 0.4223 - acc: 0.8315\n",
      "Epoch 11/15\n",
      "801/801 [==============================] - 0s 454us/step - loss: 0.4279 - acc: 0.8252\n",
      "Epoch 12/15\n",
      "801/801 [==============================] - 0s 410us/step - loss: 0.4143 - acc: 0.8265\n",
      "Epoch 13/15\n",
      "801/801 [==============================] - 0s 405us/step - loss: 0.4222 - acc: 0.8327\n",
      "Epoch 14/15\n",
      "801/801 [==============================] - 0s 395us/step - loss: 0.4286 - acc: 0.8252\n",
      "Epoch 15/15\n",
      "801/801 [==============================] - 0s 449us/step - loss: 0.4208 - acc: 0.8252\n",
      "90/90 [==============================] - 2s 20ms/step\n",
      "Epoch 1/15\n",
      "802/802 [==============================] - 5s 6ms/step - loss: 0.5896 - acc: 0.7032\n",
      "Epoch 2/15\n",
      "802/802 [==============================] - 0s 369us/step - loss: 0.4892 - acc: 0.7880\n",
      "Epoch 3/15\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4727 - acc: 0.8005\n",
      "Epoch 4/15\n",
      "802/802 [==============================] - 0s 359us/step - loss: 0.4365 - acc: 0.8192\n",
      "Epoch 5/15\n",
      "802/802 [==============================] - 0s 354us/step - loss: 0.4562 - acc: 0.8192\n",
      "Epoch 6/15\n",
      "802/802 [==============================] - 0s 414us/step - loss: 0.4480 - acc: 0.8192\n",
      "Epoch 7/15\n",
      "802/802 [==============================] - 0s 429us/step - loss: 0.4414 - acc: 0.8204\n",
      "Epoch 8/15\n",
      "802/802 [==============================] - 0s 379us/step - loss: 0.4386 - acc: 0.8217\n",
      "Epoch 9/15\n",
      "802/802 [==============================] - 0s 364us/step - loss: 0.4308 - acc: 0.8254\n",
      "Epoch 10/15\n",
      "802/802 [==============================] - 0s 374us/step - loss: 0.4345 - acc: 0.8317\n",
      "Epoch 11/15\n",
      "802/802 [==============================] - 0s 369us/step - loss: 0.4119 - acc: 0.8304\n",
      "Epoch 12/15\n",
      "802/802 [==============================] - 0s 364us/step - loss: 0.4267 - acc: 0.8342\n",
      "Epoch 13/15\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4342 - acc: 0.8317\n",
      "Epoch 14/15\n",
      "802/802 [==============================] - 0s 364us/step - loss: 0.4422 - acc: 0.8267\n",
      "Epoch 15/15\n",
      "802/802 [==============================] - 0s 379us/step - loss: 0.4232 - acc: 0.8142\n",
      "89/89 [==============================] - 2s 20ms/step\n",
      "Epoch 1/15\n",
      "802/802 [==============================] - 5s 6ms/step - loss: 0.5974 - acc: 0.6409\n",
      "Epoch 2/15\n",
      "802/802 [==============================] - 0s 339us/step - loss: 0.4845 - acc: 0.7855\n",
      "Epoch 3/15\n",
      "802/802 [==============================] - 0s 409us/step - loss: 0.4568 - acc: 0.8267\n",
      "Epoch 4/15\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4496 - acc: 0.8304\n",
      "Epoch 5/15\n",
      "802/802 [==============================] - 0s 439us/step - loss: 0.4310 - acc: 0.8329\n",
      "Epoch 6/15\n",
      "802/802 [==============================] - 0s 459us/step - loss: 0.4391 - acc: 0.8229\n",
      "Epoch 7/15\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4456 - acc: 0.8292\n",
      "Epoch 8/15\n",
      "802/802 [==============================] - 0s 484us/step - loss: 0.4319 - acc: 0.8367\n",
      "Epoch 9/15\n",
      "802/802 [==============================] - 0s 509us/step - loss: 0.4356 - acc: 0.8466\n",
      "Epoch 10/15\n",
      "802/802 [==============================] - 0s 464us/step - loss: 0.4238 - acc: 0.8342\n",
      "Epoch 11/15\n",
      "802/802 [==============================] - 0s 499us/step - loss: 0.4222 - acc: 0.8379\n",
      "Epoch 12/15\n",
      "802/802 [==============================] - 0s 499us/step - loss: 0.4187 - acc: 0.8429\n",
      "Epoch 13/15\n",
      "802/802 [==============================] - 0s 494us/step - loss: 0.4124 - acc: 0.8342\n",
      "Epoch 14/15\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4205 - acc: 0.8292\n",
      "Epoch 15/15\n",
      "802/802 [==============================] - 0s 509us/step - loss: 0.4234 - acc: 0.8304\n",
      "89/89 [==============================] - 2s 22ms/step\n",
      "Epoch 1/15\n",
      "802/802 [==============================] - 5s 6ms/step - loss: 0.5825 - acc: 0.7032\n",
      "Epoch 2/15\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4723 - acc: 0.7955\n",
      "Epoch 3/15\n",
      "802/802 [==============================] - 0s 429us/step - loss: 0.4609 - acc: 0.8167\n",
      "Epoch 4/15\n",
      "802/802 [==============================] - 0s 534us/step - loss: 0.4384 - acc: 0.8242\n",
      "Epoch 5/15\n",
      "802/802 [==============================] - 0s 404us/step - loss: 0.4314 - acc: 0.8279\n",
      "Epoch 6/15\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4219 - acc: 0.8329\n",
      "Epoch 7/15\n",
      "802/802 [==============================] - 0s 489us/step - loss: 0.4213 - acc: 0.8292\n",
      "Epoch 8/15\n",
      "802/802 [==============================] - 0s 444us/step - loss: 0.4042 - acc: 0.8254\n",
      "Epoch 9/15\n",
      "802/802 [==============================] - 0s 459us/step - loss: 0.4074 - acc: 0.8429\n",
      "Epoch 10/15\n",
      "802/802 [==============================] - 0s 459us/step - loss: 0.4088 - acc: 0.8379\n",
      "Epoch 11/15\n",
      "802/802 [==============================] - 0s 439us/step - loss: 0.3923 - acc: 0.8441\n",
      "Epoch 12/15\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4106 - acc: 0.8367\n",
      "Epoch 13/15\n",
      "802/802 [==============================] - 0s 399us/step - loss: 0.4032 - acc: 0.8367\n",
      "Epoch 14/15\n",
      "802/802 [==============================] - 0s 434us/step - loss: 0.4049 - acc: 0.8429\n",
      "Epoch 15/15\n",
      "802/802 [==============================] - 0s 419us/step - loss: 0.3865 - acc: 0.8429\n",
      "89/89 [==============================] - 2s 22ms/step\n",
      "Epoch 1/15\n",
      "802/802 [==============================] - 5s 6ms/step - loss: 0.5882 - acc: 0.6845\n",
      "Epoch 2/15\n",
      "802/802 [==============================] - 0s 494us/step - loss: 0.4778 - acc: 0.7993\n",
      "Epoch 3/15\n",
      "802/802 [==============================] - 0s 499us/step - loss: 0.4577 - acc: 0.7905\n",
      "Epoch 4/15\n",
      "802/802 [==============================] - 0s 419us/step - loss: 0.4559 - acc: 0.8292\n",
      "Epoch 5/15\n",
      "802/802 [==============================] - 0s 299us/step - loss: 0.4429 - acc: 0.8080\n",
      "Epoch 6/15\n",
      "802/802 [==============================] - 0s 284us/step - loss: 0.4569 - acc: 0.8204\n",
      "Epoch 7/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "802/802 [==============================] - 0s 284us/step - loss: 0.4448 - acc: 0.8155\n",
      "Epoch 8/15\n",
      "802/802 [==============================] - 0s 289us/step - loss: 0.4295 - acc: 0.8279\n",
      "Epoch 9/15\n",
      "802/802 [==============================] - 0s 289us/step - loss: 0.4273 - acc: 0.8167\n",
      "Epoch 10/15\n",
      "802/802 [==============================] - 0s 294us/step - loss: 0.4159 - acc: 0.8329\n",
      "Epoch 11/15\n",
      "802/802 [==============================] - 0s 324us/step - loss: 0.4050 - acc: 0.8204\n",
      "Epoch 12/15\n",
      "802/802 [==============================] - 0s 294us/step - loss: 0.4312 - acc: 0.8292\n",
      "Epoch 13/15\n",
      "802/802 [==============================] - 0s 294us/step - loss: 0.4210 - acc: 0.8242\n",
      "Epoch 14/15\n",
      "802/802 [==============================] - 0s 274us/step - loss: 0.4256 - acc: 0.8354\n",
      "Epoch 15/15\n",
      "802/802 [==============================] - 0s 344us/step - loss: 0.4163 - acc: 0.8304\n",
      "89/89 [==============================] - 2s 25ms/step\n",
      "Epoch 1/15\n",
      "802/802 [==============================] - 5s 6ms/step - loss: 0.5917 - acc: 0.6870\n",
      "Epoch 2/15\n",
      "802/802 [==============================] - 0s 399us/step - loss: 0.4777 - acc: 0.8030 0s - loss: 0.5208 - acc: 0.7\n",
      "Epoch 3/15\n",
      "802/802 [==============================] - 0s 394us/step - loss: 0.4586 - acc: 0.8292\n",
      "Epoch 4/15\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4500 - acc: 0.8229\n",
      "Epoch 5/15\n",
      "802/802 [==============================] - 0s 374us/step - loss: 0.4416 - acc: 0.8254\n",
      "Epoch 6/15\n",
      "802/802 [==============================] - 0s 394us/step - loss: 0.4388 - acc: 0.8254\n",
      "Epoch 7/15\n",
      "802/802 [==============================] - 0s 359us/step - loss: 0.4314 - acc: 0.8229\n",
      "Epoch 8/15\n",
      "802/802 [==============================] - 0s 399us/step - loss: 0.4311 - acc: 0.8329\n",
      "Epoch 9/15\n",
      "802/802 [==============================] - 0s 404us/step - loss: 0.4315 - acc: 0.8329\n",
      "Epoch 10/15\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4322 - acc: 0.8354\n",
      "Epoch 11/15\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4238 - acc: 0.8367\n",
      "Epoch 12/15\n",
      "802/802 [==============================] - 0s 369us/step - loss: 0.4117 - acc: 0.8379\n",
      "Epoch 13/15\n",
      "802/802 [==============================] - 0s 424us/step - loss: 0.4238 - acc: 0.8292\n",
      "Epoch 14/15\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4123 - acc: 0.8292\n",
      "Epoch 15/15\n",
      "802/802 [==============================] - 0s 399us/step - loss: 0.3985 - acc: 0.8367\n",
      "89/89 [==============================] - 2s 22ms/step\n",
      "Epoch 1/15\n",
      "802/802 [==============================] - 5s 6ms/step - loss: 0.5839 - acc: 0.7007\n",
      "Epoch 2/15\n",
      "802/802 [==============================] - 0s 354us/step - loss: 0.4769 - acc: 0.7918\n",
      "Epoch 3/15\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4683 - acc: 0.8142\n",
      "Epoch 4/15\n",
      "802/802 [==============================] - 0s 449us/step - loss: 0.4677 - acc: 0.8180\n",
      "Epoch 5/15\n",
      "802/802 [==============================] - 0s 464us/step - loss: 0.4521 - acc: 0.8229\n",
      "Epoch 6/15\n",
      "802/802 [==============================] - 0s 394us/step - loss: 0.4483 - acc: 0.8142\n",
      "Epoch 7/15\n",
      "802/802 [==============================] - 0s 374us/step - loss: 0.4371 - acc: 0.8192\n",
      "Epoch 8/15\n",
      "802/802 [==============================] - 0s 484us/step - loss: 0.4351 - acc: 0.8292\n",
      "Epoch 9/15\n",
      "802/802 [==============================] - 0s 399us/step - loss: 0.4357 - acc: 0.8155\n",
      "Epoch 10/15\n",
      "802/802 [==============================] - 0s 369us/step - loss: 0.4394 - acc: 0.8342\n",
      "Epoch 11/15\n",
      "802/802 [==============================] - 0s 439us/step - loss: 0.4336 - acc: 0.8279\n",
      "Epoch 12/15\n",
      "802/802 [==============================] - 0s 379us/step - loss: 0.4333 - acc: 0.8180\n",
      "Epoch 13/15\n",
      "802/802 [==============================] - 0s 359us/step - loss: 0.4274 - acc: 0.8317\n",
      "Epoch 14/15\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4366 - acc: 0.8292\n",
      "Epoch 15/15\n",
      "802/802 [==============================] - 0s 379us/step - loss: 0.4356 - acc: 0.8329\n",
      "89/89 [==============================] - 2s 22ms/step\n",
      "Epoch 1/15\n",
      "802/802 [==============================] - 5s 6ms/step - loss: 0.5786 - acc: 0.6771\n",
      "Epoch 2/15\n",
      "802/802 [==============================] - ETA: 0s - loss: 0.4787 - acc: 0.811 - 0s 349us/step - loss: 0.4747 - acc: 0.8142\n",
      "Epoch 3/15\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4675 - acc: 0.8005\n",
      "Epoch 4/15\n",
      "802/802 [==============================] - 0s 369us/step - loss: 0.4526 - acc: 0.8242\n",
      "Epoch 5/15\n",
      "802/802 [==============================] - 0s 364us/step - loss: 0.4505 - acc: 0.8242\n",
      "Epoch 6/15\n",
      "802/802 [==============================] - 0s 379us/step - loss: 0.4420 - acc: 0.8279\n",
      "Epoch 7/15\n",
      "802/802 [==============================] - 0s 374us/step - loss: 0.4351 - acc: 0.8292\n",
      "Epoch 8/15\n",
      "802/802 [==============================] - 0s 364us/step - loss: 0.4294 - acc: 0.8317\n",
      "Epoch 9/15\n",
      "802/802 [==============================] - 0s 379us/step - loss: 0.4284 - acc: 0.8304\n",
      "Epoch 10/15\n",
      "802/802 [==============================] - 0s 379us/step - loss: 0.4244 - acc: 0.8379\n",
      "Epoch 11/15\n",
      "802/802 [==============================] - 0s 384us/step - loss: 0.4416 - acc: 0.8217\n",
      "Epoch 12/15\n",
      "802/802 [==============================] - 0s 389us/step - loss: 0.4175 - acc: 0.8416\n",
      "Epoch 13/15\n",
      "802/802 [==============================] - 0s 374us/step - loss: 0.4116 - acc: 0.8404\n",
      "Epoch 14/15\n",
      "802/802 [==============================] - 0s 379us/step - loss: 0.4121 - acc: 0.8367\n",
      "Epoch 15/15\n",
      "802/802 [==============================] - 0s 379us/step - loss: 0.4106 - acc: 0.8329\n",
      "89/89 [==============================] - 2s 22ms/step\n",
      "Epoch 1/15\n",
      "803/803 [==============================] - 5s 6ms/step - loss: 0.5754 - acc: 0.7148\n",
      "Epoch 2/15\n",
      "803/803 [==============================] - 0s 558us/step - loss: 0.4908 - acc: 0.8032\n",
      "Epoch 3/15\n",
      "803/803 [==============================] - 0s 573us/step - loss: 0.4547 - acc: 0.8095\n",
      "Epoch 4/15\n",
      "803/803 [==============================] - 0s 448us/step - loss: 0.4818 - acc: 0.8070\n",
      "Epoch 5/15\n",
      "803/803 [==============================] - 0s 433us/step - loss: 0.4514 - acc: 0.8331\n",
      "Epoch 6/15\n",
      "803/803 [==============================] - 0s 418us/step - loss: 0.4405 - acc: 0.8257\n",
      "Epoch 7/15\n",
      "803/803 [==============================] - 0s 488us/step - loss: 0.4490 - acc: 0.8219\n",
      "Epoch 8/15\n",
      "803/803 [==============================] - 0s 473us/step - loss: 0.4197 - acc: 0.8356\n",
      "Epoch 9/15\n",
      "803/803 [==============================] - 0s 379us/step - loss: 0.4373 - acc: 0.8281\n",
      "Epoch 10/15\n",
      "803/803 [==============================] - 0s 488us/step - loss: 0.4443 - acc: 0.8232\n",
      "Epoch 11/15\n",
      "803/803 [==============================] - 0s 468us/step - loss: 0.4276 - acc: 0.8344\n",
      "Epoch 12/15\n",
      "803/803 [==============================] - 0s 448us/step - loss: 0.4217 - acc: 0.8294\n",
      "Epoch 13/15\n",
      "803/803 [==============================] - 0s 413us/step - loss: 0.4130 - acc: 0.8294\n",
      "Epoch 14/15\n",
      "803/803 [==============================] - 0s 478us/step - loss: 0.4252 - acc: 0.8294\n",
      "Epoch 15/15\n",
      "803/803 [==============================] - 0s 468us/step - loss: 0.4110 - acc: 0.8406\n",
      "88/88 [==============================] - 2s 23ms/step\n",
      "Results: 83.06% (2.51%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "seed = 7\n",
    "np.random.seed(seed)\n",
    "def create_smaller_2():\n",
    "  model = models.Sequential()\n",
    "  adam = optimizers.Adam(lr = 0.005)\n",
    "  model.add(layers.Dense(60,activation = 'relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), \n",
    "                         input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dropout(0.35))\n",
    "  model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "  model.add(layers.Dropout(0.20))\n",
    "  model.add(layers.Dense(1,kernel_initializer = 'normal', activation = 'sigmoid'))\n",
    "  model.compile(optimizer = adam,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller_2, epochs=15, batch_size=10, verbose=1)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "801/801 [==============================] - 6s 7ms/step - loss: 0.6895 - acc: 0.6280\n",
      "Epoch 2/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.6772 - acc: 0.6192\n",
      "Epoch 3/20\n",
      "801/801 [==============================] - 0s 65us/step - loss: 0.6519 - acc: 0.6392\n",
      "Epoch 4/20\n",
      "801/801 [==============================] - 0s 65us/step - loss: 0.6138 - acc: 0.6654\n",
      "Epoch 5/20\n",
      "801/801 [==============================] - 0s 65us/step - loss: 0.5727 - acc: 0.7141\n",
      "Epoch 6/20\n",
      "801/801 [==============================] - 0s 70us/step - loss: 0.5212 - acc: 0.7765\n",
      "Epoch 7/20\n",
      "801/801 [==============================] - 0s 70us/step - loss: 0.4908 - acc: 0.7853\n",
      "Epoch 8/20\n",
      "801/801 [==============================] - 0s 65us/step - loss: 0.5011 - acc: 0.7978\n",
      "Epoch 9/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4707 - acc: 0.8015\n",
      "Epoch 10/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4762 - acc: 0.7853\n",
      "Epoch 11/20\n",
      "801/801 [==============================] - 0s 70us/step - loss: 0.4437 - acc: 0.8177\n",
      "Epoch 12/20\n",
      "801/801 [==============================] - 0s 75us/step - loss: 0.4707 - acc: 0.8127\n",
      "Epoch 13/20\n",
      "801/801 [==============================] - 0s 80us/step - loss: 0.4439 - acc: 0.8265\n",
      "Epoch 14/20\n",
      "801/801 [==============================] - 0s 70us/step - loss: 0.4437 - acc: 0.8265\n",
      "Epoch 15/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4471 - acc: 0.8215\n",
      "Epoch 16/20\n",
      "801/801 [==============================] - 0s 54us/step - loss: 0.4343 - acc: 0.8165\n",
      "Epoch 17/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4360 - acc: 0.8277\n",
      "Epoch 18/20\n",
      "801/801 [==============================] - 0s 65us/step - loss: 0.4407 - acc: 0.8227\n",
      "Epoch 19/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4362 - acc: 0.8290\n",
      "Epoch 20/20\n",
      "801/801 [==============================] - 0s 65us/step - loss: 0.4288 - acc: 0.8177\n",
      "90/90 [==============================] - 3s 29ms/step\n",
      "Epoch 1/20\n",
      "801/801 [==============================] - 6s 7ms/step - loss: 0.6919 - acc: 0.5705\n",
      "Epoch 2/20\n",
      "801/801 [==============================] - 0s 50us/step - loss: 0.6847 - acc: 0.6167\n",
      "Epoch 3/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.6691 - acc: 0.6167\n",
      "Epoch 4/20\n",
      "801/801 [==============================] - 0s 55us/step - loss: 0.6404 - acc: 0.6317\n",
      "Epoch 5/20\n",
      "801/801 [==============================] - 0s 65us/step - loss: 0.5941 - acc: 0.6879\n",
      "Epoch 6/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.5573 - acc: 0.7179\n",
      "Epoch 7/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.5207 - acc: 0.7865\n",
      "Epoch 8/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.5028 - acc: 0.7878\n",
      "Epoch 9/20\n",
      "801/801 [==============================] - 0s 65us/step - loss: 0.4881 - acc: 0.7915\n",
      "Epoch 10/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4804 - acc: 0.8065\n",
      "Epoch 11/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4790 - acc: 0.7953\n",
      "Epoch 12/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4687 - acc: 0.8052\n",
      "Epoch 13/20\n",
      "801/801 [==============================] - 0s 65us/step - loss: 0.4557 - acc: 0.8152\n",
      "Epoch 14/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4702 - acc: 0.8002\n",
      "Epoch 15/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4603 - acc: 0.8152\n",
      "Epoch 16/20\n",
      "801/801 [==============================] - 0s 60us/step - loss: 0.4642 - acc: 0.8077\n",
      "Epoch 17/20\n",
      "801/801 [==============================] - 0s 65us/step - loss: 0.4570 - acc: 0.8152\n",
      "Epoch 18/20\n",
      "801/801 [==============================] - 0s 70us/step - loss: 0.4515 - acc: 0.8177\n",
      "Epoch 19/20\n",
      "801/801 [==============================] - 0s 75us/step - loss: 0.4472 - acc: 0.8227\n",
      "Epoch 20/20\n",
      "801/801 [==============================] - 0s 70us/step - loss: 0.4293 - acc: 0.8202\n",
      "90/90 [==============================] - 2s 28ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 7ms/step - loss: 0.6892 - acc: 0.6122\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.6718 - acc: 0.6160\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.6387 - acc: 0.6160\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.6051 - acc: 0.6160\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.5702 - acc: 0.6322\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5402 - acc: 0.7007\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.5339 - acc: 0.7756\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5145 - acc: 0.7781\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5096 - acc: 0.7893\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4938 - acc: 0.8130\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4761 - acc: 0.8005\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4821 - acc: 0.8130\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4725 - acc: 0.8130\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4579 - acc: 0.8092\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 56us/step - loss: 0.4505 - acc: 0.8167\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4586 - acc: 0.8155\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4361 - acc: 0.8229\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4620 - acc: 0.8192\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4473 - acc: 0.8192\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 80us/step - loss: 0.4263 - acc: 0.8204\n",
      "89/89 [==============================] - 2s 28ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 7ms/step - loss: 0.6925 - acc: 0.5661\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.6879 - acc: 0.6172\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.6784 - acc: 0.6185\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.6566 - acc: 0.6384\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.6128 - acc: 0.6758\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5559 - acc: 0.7257\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5027 - acc: 0.7643\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4835 - acc: 0.7855\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4832 - acc: 0.7955\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4771 - acc: 0.8092\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 75us/step - loss: 0.4554 - acc: 0.8167\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 75us/step - loss: 0.4579 - acc: 0.8130\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4473 - acc: 0.8317\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4537 - acc: 0.8217\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4446 - acc: 0.8279\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4442 - acc: 0.8304\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 75us/step - loss: 0.4529 - acc: 0.8229\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4282 - acc: 0.8292\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4325 - acc: 0.8367\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 75us/step - loss: 0.4434 - acc: 0.8254\n",
      "89/89 [==============================] - 3s 29ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 8ms/step - loss: 0.6885 - acc: 0.5985\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.6732 - acc: 0.6160\n",
      "Epoch 3/20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "802/802 [==============================] - 0s 65us/step - loss: 0.6460 - acc: 0.6172\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.6036 - acc: 0.6309\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 75us/step - loss: 0.5669 - acc: 0.6958\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.5394 - acc: 0.7282\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5136 - acc: 0.7843\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5020 - acc: 0.7880\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4819 - acc: 0.8055\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4698 - acc: 0.8092\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4706 - acc: 0.8092\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4447 - acc: 0.8279\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 59us/step - loss: 0.4652 - acc: 0.8042\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 75us/step - loss: 0.4495 - acc: 0.8267\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 75us/step - loss: 0.4422 - acc: 0.8254\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4465 - acc: 0.8304\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4381 - acc: 0.8242\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4477 - acc: 0.8229\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4323 - acc: 0.8479\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4195 - acc: 0.8392\n",
      "89/89 [==============================] - 2s 28ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 7ms/step - loss: 0.6908 - acc: 0.5973\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.6811 - acc: 0.6160\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.6600 - acc: 0.6160\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.6237 - acc: 0.6272\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5804 - acc: 0.6608\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.5495 - acc: 0.7170\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5259 - acc: 0.7531\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5076 - acc: 0.7618\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.4993 - acc: 0.7855\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4913 - acc: 0.7955\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4733 - acc: 0.8017\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4698 - acc: 0.7968\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4741 - acc: 0.8067\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4569 - acc: 0.8092\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4663 - acc: 0.8017\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.4547 - acc: 0.8117\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.4481 - acc: 0.8167\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4404 - acc: 0.8267\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 58us/step - loss: 0.4360 - acc: 0.8204\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4474 - acc: 0.8204\n",
      "89/89 [==============================] - 3s 30ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 8ms/step - loss: 0.6920 - acc: 0.5648\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.6850 - acc: 0.6160\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.6700 - acc: 0.6172\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.6438 - acc: 0.6222\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.6008 - acc: 0.6746\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.5637 - acc: 0.7145\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.5199 - acc: 0.7656\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.5042 - acc: 0.7843\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4818 - acc: 0.8042\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4696 - acc: 0.8080\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4484 - acc: 0.8092\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.4695 - acc: 0.8067\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4636 - acc: 0.8130\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4476 - acc: 0.8180\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 75us/step - loss: 0.4493 - acc: 0.8167\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 75us/step - loss: 0.4244 - acc: 0.8229\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4476 - acc: 0.8180\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4602 - acc: 0.8229\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4453 - acc: 0.8229\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4390 - acc: 0.8229\n",
      "89/89 [==============================] - 3s 32ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 8ms/step - loss: 0.6877 - acc: 0.6135\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.6616 - acc: 0.6172\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.6209 - acc: 0.6160\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.5900 - acc: 0.6459\n",
      "Epoch 5/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5585 - acc: 0.6970\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.5341 - acc: 0.7756\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.5275 - acc: 0.7731\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.5110 - acc: 0.7706\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4885 - acc: 0.8030\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4818 - acc: 0.7930\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4940 - acc: 0.7868\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4796 - acc: 0.7968\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4763 - acc: 0.7993\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4763 - acc: 0.7968\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4728 - acc: 0.8055\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4655 - acc: 0.8167\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4584 - acc: 0.8092\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4500 - acc: 0.8117\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4419 - acc: 0.8180\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4562 - acc: 0.8067\n",
      "89/89 [==============================] - 3s 30ms/step\n",
      "Epoch 1/20\n",
      "802/802 [==============================] - 6s 8ms/step - loss: 0.6902 - acc: 0.6035\n",
      "Epoch 2/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.6791 - acc: 0.6172\n",
      "Epoch 3/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.6589 - acc: 0.6247\n",
      "Epoch 4/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.6210 - acc: 0.6496\n",
      "Epoch 5/20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "802/802 [==============================] - 0s 60us/step - loss: 0.5799 - acc: 0.6808\n",
      "Epoch 6/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.5427 - acc: 0.7319\n",
      "Epoch 7/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.5066 - acc: 0.7855\n",
      "Epoch 8/20\n",
      "802/802 [==============================] - 0s 63us/step - loss: 0.4957 - acc: 0.7756\n",
      "Epoch 9/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4885 - acc: 0.7930\n",
      "Epoch 10/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4833 - acc: 0.8092\n",
      "Epoch 11/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4622 - acc: 0.8030\n",
      "Epoch 12/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4653 - acc: 0.8130\n",
      "Epoch 13/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.4502 - acc: 0.8142\n",
      "Epoch 14/20\n",
      "802/802 [==============================] - 0s 65us/step - loss: 0.4587 - acc: 0.8192\n",
      "Epoch 15/20\n",
      "802/802 [==============================] - 0s 55us/step - loss: 0.4554 - acc: 0.8130\n",
      "Epoch 16/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4464 - acc: 0.8167\n",
      "Epoch 17/20\n",
      "802/802 [==============================] - 0s 60us/step - loss: 0.4492 - acc: 0.8217\n",
      "Epoch 18/20\n",
      "802/802 [==============================] - 0s 80us/step - loss: 0.4450 - acc: 0.8254\n",
      "Epoch 19/20\n",
      "802/802 [==============================] - 0s 80us/step - loss: 0.4410 - acc: 0.8304\n",
      "Epoch 20/20\n",
      "802/802 [==============================] - 0s 70us/step - loss: 0.4266 - acc: 0.8367\n",
      "89/89 [==============================] - 3s 31ms/step\n",
      "Epoch 1/20\n",
      "803/803 [==============================] - 6s 8ms/step - loss: 0.6914 - acc: 0.5604\n",
      "Epoch 2/20\n",
      "803/803 [==============================] - 0s 55us/step - loss: 0.6816 - acc: 0.6264\n",
      "Epoch 3/20\n",
      "803/803 [==============================] - 0s 65us/step - loss: 0.6619 - acc: 0.6301\n",
      "Epoch 4/20\n",
      "803/803 [==============================] - 0s 60us/step - loss: 0.6311 - acc: 0.6687\n",
      "Epoch 5/20\n",
      "803/803 [==============================] - 0s 60us/step - loss: 0.5876 - acc: 0.6986\n",
      "Epoch 6/20\n",
      "803/803 [==============================] - 0s 57us/step - loss: 0.5368 - acc: 0.7659\n",
      "Epoch 7/20\n",
      "803/803 [==============================] - 0s 65us/step - loss: 0.4939 - acc: 0.7858\n",
      "Epoch 8/20\n",
      "803/803 [==============================] - 0s 56us/step - loss: 0.4899 - acc: 0.7908\n",
      "Epoch 9/20\n",
      "803/803 [==============================] - 0s 60us/step - loss: 0.4669 - acc: 0.8082\n",
      "Epoch 10/20\n",
      "803/803 [==============================] - 0s 65us/step - loss: 0.4561 - acc: 0.8169\n",
      "Epoch 11/20\n",
      "803/803 [==============================] - 0s 65us/step - loss: 0.4598 - acc: 0.8182\n",
      "Epoch 12/20\n",
      "803/803 [==============================] - 0s 60us/step - loss: 0.4591 - acc: 0.8132\n",
      "Epoch 13/20\n",
      "803/803 [==============================] - 0s 70us/step - loss: 0.4578 - acc: 0.8095\n",
      "Epoch 14/20\n",
      "803/803 [==============================] - 0s 60us/step - loss: 0.4571 - acc: 0.8232\n",
      "Epoch 15/20\n",
      "803/803 [==============================] - 0s 75us/step - loss: 0.4447 - acc: 0.8169\n",
      "Epoch 16/20\n",
      "803/803 [==============================] - 0s 80us/step - loss: 0.4491 - acc: 0.8182\n",
      "Epoch 17/20\n",
      "803/803 [==============================] - 0s 75us/step - loss: 0.4467 - acc: 0.8257\n",
      "Epoch 18/20\n",
      "803/803 [==============================] - 0s 85us/step - loss: 0.4463 - acc: 0.8257\n",
      "Epoch 19/20\n",
      "803/803 [==============================] - 0s 80us/step - loss: 0.4345 - acc: 0.8207\n",
      "Epoch 20/20\n",
      "803/803 [==============================] - 0s 65us/step - loss: 0.4480 - acc: 0.8294\n",
      "88/88 [==============================] - 3s 29ms/step\n",
      "Results: 82.60% (3.98%)\n"
     ]
    }
   ],
   "source": [
    "# smaller model\n",
    "seed = 7\n",
    "np.random.seed(seed)\n",
    "def create_smaller_2():\n",
    "  model = models.Sequential()\n",
    "  adam = optimizers.Adam(lr = 0.003)\n",
    "  model.add(layers.Dense(60,activation = 'relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), \n",
    "                         input_shape =(training_predictors.shape[1],)))\n",
    "  model.add(layers.Dropout(0.50))\n",
    "  model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "  model.add(layers.Dropout(0.20))\n",
    "  model.add(layers.Dense(1,kernel_initializer = 'normal', activation = 'sigmoid'))\n",
    "  model.compile(optimizer = adam,loss = 'binary_crossentropy',metrics=['accuracy'] )\n",
    "  return model\n",
    "estimator = KerasClassifier(build_fn=create_smaller_2, epochs=20, batch_size=128, verbose=1)\n",
    "kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(estimator, training_predictors, training_target, cv=kfold)\n",
    "print(\"Results: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x28c3cd8ea58>"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#USing 83.39% accuracy model\n",
    "np.random.seed(seed)\n",
    "model = models.Sequential()\n",
    "adam = optimizers.Adam(lr = 0.003)\n",
    "model.add(layers.Dense(60,activation = 'relu',kernel_initializer = 'normal',kernel_constraint = maxnorm(3), \n",
    "                         input_shape =(training_predictors.shape[1],)))\n",
    "model.add(layers.Dropout(0.50))\n",
    "model.add(layers.Dense(10,kernel_initializer = 'normal',kernel_constraint = maxnorm(3), activation = 'relu'))\n",
    "model.add(layers.Dropout(0.20))\n",
    "model.add(layers.Dense(1,kernel_initializer = 'normal', activation = 'sigmoid'))\n",
    "model.compile(optimizer = adam,loss = 'binary_crossentropy',metrics=['binary_accuracy'] )\n",
    "model.fit(training_predictors,\n",
    "          training_target,\n",
    "                    \n",
    "                    epochs=20,\n",
    "                    batch_size=5,\n",
    "                    verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "answers = model.predict(testing_predictors).round().astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_file = pd.DataFrame({'PassengerId':passengerId, 'Survived':answers.ravel()})\n",
    "final_file.to_csv('deepResult.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}