credit_risk_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "efb934d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import modules and dependencies\n",
    "from warnings import simplefilter\n",
    "simplefilter(action='ignore', category=FutureWarning)\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report\n",
    "from imblearn.metrics import classification_report_imbalanced\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import accuracy_score, balanced_accuracy_score, confusion_matrix\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fd98d7c",
   "metadata": {},
   "source": [
    "## Split the Data into Training and Testing Sets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5d046d5",
   "metadata": {},
   "source": [
    "### Step 1: Read the `lending_data.csv` data from the `Resources` folder into a Pandas DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bd81e1f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_size</th>\n",
       "      <th>interest_rate</th>\n",
       "      <th>borrower_income</th>\n",
       "      <th>debt_to_income</th>\n",
       "      <th>num_of_accounts</th>\n",
       "      <th>derogatory_marks</th>\n",
       "      <th>total_debt</th>\n",
       "      <th>loan_status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10700.0</td>\n",
       "      <td>7.672</td>\n",
       "      <td>52800</td>\n",
       "      <td>0.431818</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>22800</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8400.0</td>\n",
       "      <td>6.692</td>\n",
       "      <td>43600</td>\n",
       "      <td>0.311927</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13600</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9000.0</td>\n",
       "      <td>6.963</td>\n",
       "      <td>46100</td>\n",
       "      <td>0.349241</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>16100</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10700.0</td>\n",
       "      <td>7.664</td>\n",
       "      <td>52700</td>\n",
       "      <td>0.430740</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>22700</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10800.0</td>\n",
       "      <td>7.698</td>\n",
       "      <td>53000</td>\n",
       "      <td>0.433962</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>23000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_size  interest_rate  borrower_income  debt_to_income  num_of_accounts  \\\n",
       "0    10700.0          7.672            52800        0.431818                5   \n",
       "1     8400.0          6.692            43600        0.311927                3   \n",
       "2     9000.0          6.963            46100        0.349241                3   \n",
       "3    10700.0          7.664            52700        0.430740                5   \n",
       "4    10800.0          7.698            53000        0.433962                5   \n",
       "\n",
       "   derogatory_marks  total_debt  loan_status  \n",
       "0                 1       22800            0  \n",
       "1                 0       13600            0  \n",
       "2                 0       16100            0  \n",
       "3                 1       22700            0  \n",
       "4                 1       23000            0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read the lending_data.csv data into a Pandas DataFrame\n",
    "csv_path = 'Resources/lending_data.csv'\n",
    "lending_df = pd.read_csv(csv_path)\n",
    "lending_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "456fa1d5",
   "metadata": {},
   "source": [
    "### Step 2: Create the labels set (`y`)  from the “loan_status” column, and then create the features (`X`) DataFrame from the remaining columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "87065677",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create features set, X, and target labels, y\n",
    "y = lending_df['loan_status']\n",
    "X = lending_df.drop(columns='loan_status')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5dc37fef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_size</th>\n",
       "      <th>interest_rate</th>\n",
       "      <th>borrower_income</th>\n",
       "      <th>debt_to_income</th>\n",
       "      <th>num_of_accounts</th>\n",
       "      <th>derogatory_marks</th>\n",
       "      <th>total_debt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10700.0</td>\n",
       "      <td>7.672</td>\n",
       "      <td>52800</td>\n",
       "      <td>0.431818</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>22800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8400.0</td>\n",
       "      <td>6.692</td>\n",
       "      <td>43600</td>\n",
       "      <td>0.311927</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9000.0</td>\n",
       "      <td>6.963</td>\n",
       "      <td>46100</td>\n",
       "      <td>0.349241</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>16100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10700.0</td>\n",
       "      <td>7.664</td>\n",
       "      <td>52700</td>\n",
       "      <td>0.430740</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>22700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10800.0</td>\n",
       "      <td>7.698</td>\n",
       "      <td>53000</td>\n",
       "      <td>0.433962</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>23000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_size  interest_rate  borrower_income  debt_to_income  num_of_accounts  \\\n",
       "0    10700.0          7.672            52800        0.431818                5   \n",
       "1     8400.0          6.692            43600        0.311927                3   \n",
       "2     9000.0          6.963            46100        0.349241                3   \n",
       "3    10700.0          7.664            52700        0.430740                5   \n",
       "4    10800.0          7.698            53000        0.433962                5   \n",
       "\n",
       "   derogatory_marks  total_debt  \n",
       "0                 1       22800  \n",
       "1                 0       13600  \n",
       "2                 0       16100  \n",
       "3                 1       22700  \n",
       "4                 1       23000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Review the X variable DataFrame\n",
    "X[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "88a2d3bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "3    0\n",
       "4    0\n",
       "Name: loan_status, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Review the y variable Series\n",
    "y[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdae6739",
   "metadata": {},
   "source": [
    "### Step 3: Check the balance of the labels variable (`y`) by using the `value_counts` function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "17fcd99e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    75036\n",
       "1     2500\n",
       "Name: loan_status, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the balance of our target values\n",
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0e04828",
   "metadata": {},
   "source": [
    "### Step 4: Split the data into training and testing datasets by using `train_test_split`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b4cdc11b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into training and testing datasets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49b9ee04",
   "metadata": {},
   "source": [
    "## Create a Logistic Regression Model with the Original Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bc7be44",
   "metadata": {},
   "source": [
    "###  Step 1: Fit a logistic regression model by using the training data (`X_train` and `y_train`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "51f0322e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate the Logistic Regression model\n",
    "# Assign a random_state parameter of 1 to the model\n",
    "logistic_regression_model = LogisticRegression(random_state=1)\n",
    "\n",
    "# Fit the model using training data\n",
    "lr_model = logistic_regression_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d4a5a16",
   "metadata": {},
   "source": [
    "### Step 2: Save the predictions on the testing data labels by using the testing feature data (`X_test`) and the fitted model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8accd734",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Score: 0.9921240885954051\n",
      "Testing Data Score: 0.9918489475856377\n"
     ]
    }
   ],
   "source": [
    "# Make a prediction using the testing data\n",
    "# Generate training predictions\n",
    "training_predictions = lr_model.predict(X_train)\n",
    "print(f\"Training Data Score: {lr_model.score(X_train, y_train)}\")\n",
    "\n",
    "# Generate testing predictions\n",
    "testing_predictions = logistic_regression_model.predict(X_test)\n",
    "print(f\"Testing Data Score: {lr_model.score(X_test, y_test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5281c831",
   "metadata": {},
   "source": [
    "### Step 3: Evaluate the model’s performance by doing the following:\n",
    "\n",
    "* Calculate the accuracy score of the model.\n",
    "\n",
    "* Generate a confusion matrix.\n",
    "\n",
    "* Print the classification report."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "93a22fd6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Score: 0.9918489475856377\n",
      "Balanced Accuracy Score: 0.9520479254722232\n"
     ]
    }
   ],
   "source": [
    "# Print the accuracy and balanced_accuracy score of the model\n",
    "print(f\"Accuracy Score: {accuracy_score(y_test, logistic_regression_model.predict(X_test))}\")\n",
    "print(f\"Balanced Accuracy Score: {balanced_accuracy_score(y_test, logistic_regression_model.predict(X_test))}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "97c6ab67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[55994   277]\n",
      " [  181  1700]]\n"
     ]
    }
   ],
   "source": [
    "# Create and save the confusion matrix for the TRAINING data\n",
    "training_matrix = confusion_matrix(y_train, training_predictions)\n",
    "\n",
    "# Print the confusion matrix for the training data\n",
    "print(training_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "83b30c95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Predicted Healthy Loans (low-risk)</th>\n",
       "      <th>Predicted High-Risk Loans (high-risk)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Actual Healthy Loans (low-risk)</th>\n",
       "      <td>55994</td>\n",
       "      <td>277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Actual High-Risk Loans (high-risk)</th>\n",
       "      <td>181</td>\n",
       "      <td>1700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Predicted Healthy Loans (low-risk)  \\\n",
       "Actual Healthy Loans (low-risk)                                  55994   \n",
       "Actual High-Risk Loans (high-risk)                                 181   \n",
       "\n",
       "                                    Predicted High-Risk Loans (high-risk)  \n",
       "Actual Healthy Loans (low-risk)                                       277  \n",
       "Actual High-Risk Loans (high-risk)                                   1700  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert confusion matrix for the TRAINING model into a DataFrame to review; we see from above it is imbalanced  \n",
    "cm_imbalanced_train = confusion_matrix(y_train, training_predictions)\n",
    "cm_imbalanced_df = pd.DataFrame(cm_imbalanced_train, \n",
    "                                index = ['Actual Healthy Loans (low-risk)', 'Actual High-Risk Loans (high-risk)'], \n",
    "                                columns = ['Predicted Healthy Loans (low-risk)', 'Predicted High-Risk Loans (high-risk)']\n",
    "                              )\n",
    "cm_imbalanced_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81e543c9",
   "metadata": {},
   "source": [
    "**Training Model:**\n",
    "- Out of the 56,271 healthy loan statuses, the model predicted 55,994 as healthy correctly and 277 as healthy incorrectly. \n",
    "\n",
    "- Out of the 1881 high-risk loan statuses, the model predicted 181 as high-risk correctly and 1700 as high-risk incorrectly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "20177854",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[18663   102]\n",
      " [   56   563]]\n"
     ]
    }
   ],
   "source": [
    "# Create and save the confusion matrix for the TESTING data\n",
    "test_matrix = confusion_matrix(y_test, testing_predictions)\n",
    "\n",
    "# Print the confusion matrix for the testing data\n",
    "print(test_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "152dfa9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Predicted Healthy Loans (low-risk)</th>\n",
       "      <th>Predicted High-Risk Loans (high-risk)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Actual Healthy Loans (low-risk)</th>\n",
       "      <td>18663</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Actual High-Risk Loans (high-risk)</th>\n",
       "      <td>56</td>\n",
       "      <td>563</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Predicted Healthy Loans (low-risk)  \\\n",
       "Actual Healthy Loans (low-risk)                                  18663   \n",
       "Actual High-Risk Loans (high-risk)                                  56   \n",
       "\n",
       "                                    Predicted High-Risk Loans (high-risk)  \n",
       "Actual Healthy Loans (low-risk)                                       102  \n",
       "Actual High-Risk Loans (high-risk)                                    563  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert confusion matrix for the TESTING model into a DataFrame to review; we see from above it is imbalanced  \n",
    "cm_imbalanced = confusion_matrix(y_test, testing_predictions)\n",
    "cm_imbalanced_df = pd.DataFrame(cm_imbalanced, \n",
    "                                index = ['Actual Healthy Loans (low-risk)', 'Actual High-Risk Loans (high-risk)'], \n",
    "                                columns = ['Predicted Healthy Loans (low-risk)', 'Predicted High-Risk Loans (high-risk)']\n",
    "                              )\n",
    "cm_imbalanced_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8f25a57",
   "metadata": {},
   "source": [
    "**Testing Model:**\n",
    "- Out of the 18,765 healthy loan statuses, the model predicted 18,663 as healthy correctly and 102 as healthy incorrectly \n",
    "\n",
    "- Out of the 619 high-risk loan statuses, the model predicted 563 as high-risk correctly and 56 as high-risk incorrectly\n",
    "\n",
    "- There is an improvement with the testing predictions in comparison to the training predictions. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4d9a8c47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56271\n",
      "           1       0.86      0.90      0.88      1881\n",
      "\n",
      "    accuracy                           0.99     58152\n",
      "   macro avg       0.93      0.95      0.94     58152\n",
      "weighted avg       0.99      0.99      0.99     58152\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create and save the TRAINING classification report\n",
    "bal_training_report = classification_report(y_train, training_predictions)\n",
    "\n",
    "# Print the training classification report\n",
    "print(bal_training_report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "243e0eb2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   pre       rec       spe        f1       geo       iba       sup\n",
      "\n",
      "          0       1.00      1.00      0.90      1.00      0.95      0.91     56271\n",
      "          1       0.86      0.90      1.00      0.88      0.95      0.89      1881\n",
      "\n",
      "avg / total       0.99      0.99      0.91      0.99      0.95      0.91     58152\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create and save the TRAINING classification report - testing as an imbalanced set \n",
    "imbal_training_report = classification_report_imbalanced(y_train, training_predictions)\n",
    "\n",
    "print(imbal_training_report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3ed88ea0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      1.00     18765\n",
      "           1       0.85      0.91      0.88       619\n",
      "\n",
      "    accuracy                           0.99     19384\n",
      "   macro avg       0.92      0.95      0.94     19384\n",
      "weighted avg       0.99      0.99      0.99     19384\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create and save the TESTING classification report - testing as a balanced set \n",
    "bal_testing_report = classification_report(y_test, testing_predictions)\n",
    "\n",
    "# Print the testing classification report\n",
    "print(bal_testing_report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "802518ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   pre       rec       spe        f1       geo       iba       sup\n",
      "\n",
      "          0       1.00      0.99      0.91      1.00      0.95      0.91     18765\n",
      "          1       0.85      0.91      0.99      0.88      0.95      0.90       619\n",
      "\n",
      "avg / total       0.99      0.99      0.91      0.99      0.95      0.91     19384\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create and save the TESTING classification report - testing as an imbalanced set \n",
    "imbal_testing_report = classification_report_imbalanced(y_test, testing_predictions)\n",
    "\n",
    "print(imbal_testing_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "053fd51c",
   "metadata": {},
   "source": [
    "### Step 4: Answer the following question."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "299449f3",
   "metadata": {},
   "source": [
    "**Question:** How well does the logistic regression model predict both the `0` (healthy loan) and `1` (high-risk loan) labels?\n",
    "\n",
    "**Answer:** According to the balanced accuracy score of 95%, the model is performing well; however, this is likely due to an imbalanced dataset. When analyzing the dataset, the number of healthy loans (low-risk) highly outweigh the number of high-risk loans. This indicates that the model is better at predicting a loan status as healthy in comparison to a loan status as high-risk. As the dataset is imbalanced, we need to further look at the oversampled dataset. \n",
    "\n",
    "When analyzing both the balanced and imbalanced classification reports, the model is able to predict healthy loans `0` 100% of the time, with 99% of those recalled correctly. The model is only able to predict high-risk loans `1` with a precision of 85%, and of those 85% correctly identified there is a recall rate of 91%. \n",
    "\n",
    "Overall, precision and recall values are extremely improtant because the cost of misclassification can be high. A high precision score is important to minimize false positives, which can lead to a loss of potential customers. On the other hand, a high recall score is important to minimize false negatives, which can lead to significant financial losses.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c06369d",
   "metadata": {},
   "source": [
    "## Predict a Logistic Regression Model with Resampled Training Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c363abf",
   "metadata": {},
   "source": [
    "### Step 1: Use the `RandomOverSampler` module from the imbalanced-learn library to resample the data. Be sure to confirm that the labels have an equal number of data points. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9c842af0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 56271), (1, 56271)]\n"
     ]
    }
   ],
   "source": [
    "# Instantiate the random oversampler model\n",
    "# # Assign a random_state parameter of 1 to the model\n",
    "ros = RandomOverSampler(random_state=1)\n",
    "\n",
    "# Fit the original training data to the random_oversampler model\n",
    "X_train_ros, y_train_ros= ros.fit_resample(X_train, y_train)\n",
    "\n",
    "# Check the number of records after over sampling\n",
    "print(sorted(Counter(y_train_ros).items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "351ed408",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    56271\n",
       "1    56271\n",
       "Name: loan_status, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Count the distinct values of the resampled labels data\n",
    "y_train_ros.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "616fac91",
   "metadata": {},
   "source": [
    "### Step 2: Use the `LogisticRegression` classifier and the resampled data to fit the model and make predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "65a3ddac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate the Logistic Regression model\n",
    "# Assign a random_state parameter of 1 to the model\n",
    "logistic_regression_model_ros = LogisticRegression(solver='lbfgs', random_state=1)\n",
    "\n",
    "# Fit the model using resampled training data\n",
    "lr_model_ros = logistic_regression_model_ros.fit(X_train_ros, y_train_ros)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "266c50d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Score: 0.9945659650570917\n",
      "Testing Data Score: 0.9938093272802311\n"
     ]
    }
   ],
   "source": [
    "# Make a prediction using the testing data\n",
    "# Generate training predictions\n",
    "training_predictions_ros = lr_model_ros.predict(X_train_ros)\n",
    "print(f\"Training Data Score: {logistic_regression_model_ros.score(X_train, y_train)}\")\n",
    "\n",
    "# Generate testing predictions\n",
    "testing_predictions_ros = logistic_regression_model_ros.predict(X_test)\n",
    "print(f\"Testing Data Score: {logistic_regression_model_ros.score(X_test, y_test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7eca5a7d",
   "metadata": {},
   "source": [
    "### Step 3: Evaluate the model’s performance by doing the following:\n",
    "\n",
    "* Calculate the accuracy score of the model.\n",
    "\n",
    "* Generate a confusion matrix.\n",
    "\n",
    "* Print the classification report."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "488d2f2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Score: 0.9938093272802311\n",
      "Accuracy Score: 0.9936781215845847\n"
     ]
    }
   ],
   "source": [
    "# Print the accuracy and balanced_accuracy score of the model \n",
    "print(f\"Accuracy Score: {accuracy_score(y_test, logistic_regression_model_ros.predict(X_test))}\")\n",
    "print(f\"Accuracy Score: {balanced_accuracy_score(y_test, logistic_regression_model_ros.predict(X_test))}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "bf385f45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[18649   116]\n",
      " [    4   615]]\n"
     ]
    }
   ],
   "source": [
    "# Create and save the confusion matrix for the testing data\n",
    "test_matrix_ros = confusion_matrix(y_test, testing_predictions_ros)\n",
    "\n",
    "# Print the confusion matrix for the testing data\n",
    "print(test_matrix_ros)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "433ee9b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Predicted Healthy Loans (low-risk)</th>\n",
       "      <th>Predicted High-Risk Loans (high-risk)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Actual Healthy Loans (low-risk)</th>\n",
       "      <td>18649</td>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Actual High-Risk Loans (high-risk)</th>\n",
       "      <td>4</td>\n",
       "      <td>615</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Predicted Healthy Loans (low-risk)  \\\n",
       "Actual Healthy Loans (low-risk)                                  18649   \n",
       "Actual High-Risk Loans (high-risk)                                   4   \n",
       "\n",
       "                                    Predicted High-Risk Loans (high-risk)  \n",
       "Actual Healthy Loans (low-risk)                                       116  \n",
       "Actual High-Risk Loans (high-risk)                                    615  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert confusion matrix for the TESTING model into a DataFrame to review; we see from above it is imbalanced  \n",
    "cm_imbalanced = confusion_matrix(y_test, testing_predictions_ros)\n",
    "cm_imbalanced_df = pd.DataFrame(cm_imbalanced, \n",
    "                                index = ['Actual Healthy Loans (low-risk)', 'Actual High-Risk Loans (high-risk)'], \n",
    "                                columns = ['Predicted Healthy Loans (low-risk)', 'Predicted High-Risk Loans (high-risk)']\n",
    "                              )\n",
    "cm_imbalanced_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c2f9061",
   "metadata": {},
   "source": [
    "**Testing Model:**\n",
    "- Out of the 18,765 healthy loan statuses, the model predicted 18,649 as healthy correctly and 116 as healthy incorrectly \n",
    "\n",
    "- Out of the 619 high-risk loan statuses, the model predicted 4 as high-risk correctly and 615 as high-risk incorrectly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0be17d1f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99     56271\n",
      "           1       0.99      0.99      0.99     56271\n",
      "\n",
      "    accuracy                           0.99    112542\n",
      "   macro avg       0.99      0.99      0.99    112542\n",
      "weighted avg       0.99      0.99      0.99    112542\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print the classification report for the TRAINING model\n",
    "training_report_ros = classification_report(y_train_ros, training_predictions_ros)\n",
    "\n",
    "# Print the training classification report\n",
    "print(training_report_ros)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "be0f3ba5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      1.00     18765\n",
      "           1       0.84      0.99      0.91       619\n",
      "\n",
      "    accuracy                           0.99     19384\n",
      "   macro avg       0.92      0.99      0.95     19384\n",
      "weighted avg       0.99      0.99      0.99     19384\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print the classification report for the TESTING model\n",
    "testing_report_ros = classification_report(y_test, testing_predictions_ros)\n",
    "\n",
    "# Print the testing classification report\n",
    "print(testing_report_ros)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40b16331",
   "metadata": {},
   "source": [
    "### Step 4: Answer the following question"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ceb347b",
   "metadata": {},
   "source": [
    "**Question:** How well does the logistic regression model, fit with oversampled data, predict both the `0` (healthy loan) and `1` (high-risk loan) labels?\n",
    "\n",
    "**Answer:** Overall, in analyzing the results of the machine learning models, the oversampled model would be the model of choice. The oversampled model generated a balanced accuracy score of 99%, which is an improvement than the prior logistic regression model on original data of 95%. It continues to perform the same when being able to correctly identify health loans, as the precision is 100%, and the recall remains at 99%. More importantly, where the oversampled model performs better is with the recall rates for high-risk loans. Although still obtaining a similar precision as the logistic regression model on original data at 84% in comparison to 85%, of these 84% that are correctly identified at high-risk, there is a recall of 99%. This is an improvement from the 91% on the prior logistic regression model on original data. \n",
    "\n",
    "The oversampled model helps to better predict the true positives meaning that it is more effective at distinguishing high-risk loans with high recall accuracy. As well, as the number of false positives significantly decreases, the oversampled model is able to classify healthy and high-risk loans more correctly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e94c712a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}