4.b.isbankasi-bayesianoptimization-xgboost-expanded.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.008591,
     "end_time": "2020-11-07T01:36:27.270504",
     "exception": false,
     "start_time": "2020-11-07T01:36:27.261913",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "https://github.com/fmfn/BayesianOptimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-07T01:36:27.295030Z",
     "iopub.status.busy": "2020-11-07T01:36:27.293798Z",
     "iopub.status.idle": "2020-11-07T01:37:18.592438Z",
     "shell.execute_reply": "2020-11-07T01:37:18.591240Z"
    },
    "papermill": {
     "duration": 51.314731,
     "end_time": "2020-11-07T01:37:18.592592",
     "exception": false,
     "start_time": "2020-11-07T01:36:27.277861",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#read dfs# import packages\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "train_X=pd.read_csv(\"../input/vol1outputs/train_x.csv\")\n",
    "train_y=pd.read_csv(\"../input/vol1outputs/train_y.csv\")\n",
    "normalized_test_df=pd.read_csv(\"../input/vol1outputs/normalized_test_df.csv\")\n",
    "df_test1=pd.read_csv(\"../input/vol1outputs/df_test1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-07T01:37:19.093592Z",
     "iopub.status.busy": "2020-11-07T01:37:19.074059Z",
     "iopub.status.idle": "2020-11-07T01:37:19.110778Z",
     "shell.execute_reply": "2020-11-07T01:37:19.110224Z"
    },
    "papermill": {
     "duration": 0.510118,
     "end_time": "2020-11-07T01:37:19.110896",
     "exception": false,
     "start_time": "2020-11-07T01:37:18.600778",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_X=train_X.drop(columns=train_X.columns[0])\n",
    "train_y=train_y.drop(columns=train_y.columns[0])\n",
    "normalized_test_df=normalized_test_df.drop(columns=normalized_test_df.columns[0])\n",
    "df_test1=df_test1.drop(columns=df_test1.columns[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2020-11-07T01:37:19.142209Z",
     "iopub.status.busy": "2020-11-07T01:37:19.136841Z",
     "iopub.status.idle": "2020-11-07T04:40:07.250895Z",
     "shell.execute_reply": "2020-11-07T04:40:07.251715Z"
    },
    "papermill": {
     "duration": 10968.132821,
     "end_time": "2020-11-07T04:40:07.251979",
     "exception": false,
     "start_time": "2020-11-07T01:37:19.119158",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   | colsam... | colsam... | colsam... |   gamma   | learni... | max_de... | max_depth | min_ch... | n_esti... | reg_alpha | reg_la... | subsample |\n",
      "-------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m-5.445e+0\u001b[0m | \u001b[0m 0.7043  \u001b[0m | \u001b[0m 0.853   \u001b[0m | \u001b[0m 0.5001  \u001b[0m | \u001b[0m 0.2993  \u001b[0m | \u001b[0m 0.05256 \u001b[0m | \u001b[0m 0.9234  \u001b[0m | \u001b[0m 5.931   \u001b[0m | \u001b[0m 4.11    \u001b[0m | \u001b[0m 426.9   \u001b[0m | \u001b[0m 54.34   \u001b[0m | \u001b[0m 42.5    \u001b[0m | \u001b[0m 0.8358  \u001b[0m |\n",
      "| \u001b[95m 2       \u001b[0m | \u001b[95m-5.135e+0\u001b[0m | \u001b[95m 0.6002  \u001b[0m | \u001b[95m 0.9303  \u001b[0m | \u001b[95m 0.5134  \u001b[0m | \u001b[95m 0.6638  \u001b[0m | \u001b[95m 0.131   \u001b[0m | \u001b[95m 5.587   \u001b[0m | \u001b[95m 5.702   \u001b[0m | \u001b[95m 2.783   \u001b[0m | \u001b[95m 810.7   \u001b[0m | \u001b[95m 96.86   \u001b[0m | \u001b[95m 32.03   \u001b[0m | \u001b[95m 0.8392  \u001b[0m |\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m-5.18e+06\u001b[0m | \u001b[0m 0.9294  \u001b[0m | \u001b[0m 0.9384  \u001b[0m | \u001b[0m 0.5417  \u001b[0m | \u001b[0m 0.03866 \u001b[0m | \u001b[0m 0.05925 \u001b[0m | \u001b[0m 8.781   \u001b[0m | \u001b[0m 5.492   \u001b[0m | \u001b[0m 4.79    \u001b[0m | \u001b[0m 960.0   \u001b[0m | \u001b[0m 53.78   \u001b[0m | \u001b[0m 69.5    \u001b[0m | \u001b[0m 0.6546  \u001b[0m |\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m-5.273e+0\u001b[0m | \u001b[0m 0.8364  \u001b[0m | \u001b[0m 0.909   \u001b[0m | \u001b[0m 0.509   \u001b[0m | \u001b[0m 0.7426  \u001b[0m | \u001b[0m 0.2968  \u001b[0m | \u001b[0m 7.482   \u001b[0m | \u001b[0m 6.402   \u001b[0m | \u001b[0m 8.104   \u001b[0m | \u001b[0m 148.1   \u001b[0m | \u001b[0m 45.34   \u001b[0m | \u001b[0m 90.95   \u001b[0m | \u001b[0m 0.6439  \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m-5.372e+0\u001b[0m | \u001b[0m 0.641   \u001b[0m | \u001b[0m 0.5637  \u001b[0m | \u001b[0m 0.5095  \u001b[0m | \u001b[0m 0.672   \u001b[0m | \u001b[0m 0.07137 \u001b[0m | \u001b[0m 2.655   \u001b[0m | \u001b[0m 7.458   \u001b[0m | \u001b[0m 1.48    \u001b[0m | \u001b[0m 595.4   \u001b[0m | \u001b[0m 15.53   \u001b[0m | \u001b[0m 59.34   \u001b[0m | \u001b[0m 0.8429  \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m-5.399e+0\u001b[0m | \u001b[0m 0.5     \u001b[0m | \u001b[0m 0.5     \u001b[0m | \u001b[0m 0.99    \u001b[0m | \u001b[0m 0.99    \u001b[0m | \u001b[0m 0.01    \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 851.6   \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.99    \u001b[0m |\n",
      "| \u001b[95m 7       \u001b[0m | \u001b[95m-5.055e+0\u001b[0m | \u001b[95m 0.7906  \u001b[0m | \u001b[95m 0.705   \u001b[0m | \u001b[95m 0.5159  \u001b[0m | \u001b[95m 0.344   \u001b[0m | \u001b[95m 0.1805  \u001b[0m | \u001b[95m 5.486   \u001b[0m | \u001b[95m 8.241   \u001b[0m | \u001b[95m 5.296   \u001b[0m | \u001b[95m 957.2   \u001b[0m | \u001b[95m 56.35   \u001b[0m | \u001b[95m 72.97   \u001b[0m | \u001b[95m 0.6572  \u001b[0m |\n",
      "| \u001b[95m 8       \u001b[0m | \u001b[95m-1.611e+0\u001b[0m | \u001b[95m 0.5828  \u001b[0m | \u001b[95m 0.6661  \u001b[0m | \u001b[95m 0.6372  \u001b[0m | \u001b[95m 0.165   \u001b[0m | \u001b[95m 0.1033  \u001b[0m | \u001b[95m 0.3771  \u001b[0m | \u001b[95m 7.294   \u001b[0m | \u001b[95m 9.303   \u001b[0m | \u001b[95m 959.7   \u001b[0m | \u001b[95m 58.79   \u001b[0m | \u001b[95m 76.23   \u001b[0m | \u001b[95m 0.7546  \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m-5.122e+0\u001b[0m | \u001b[0m 0.9257  \u001b[0m | \u001b[0m 0.7863  \u001b[0m | \u001b[0m 0.9862  \u001b[0m | \u001b[0m 0.7668  \u001b[0m | \u001b[0m 0.225   \u001b[0m | \u001b[0m 2.559   \u001b[0m | \u001b[0m 8.389   \u001b[0m | \u001b[0m 8.247   \u001b[0m | \u001b[0m 957.4   \u001b[0m | \u001b[0m 56.95   \u001b[0m | \u001b[0m 82.69   \u001b[0m | \u001b[0m 0.9721  \u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m-5.388e+0\u001b[0m | \u001b[0m 0.5685  \u001b[0m | \u001b[0m 0.6643  \u001b[0m | \u001b[0m 0.7513  \u001b[0m | \u001b[0m 0.9547  \u001b[0m | \u001b[0m 0.1076  \u001b[0m | \u001b[0m 2.255   \u001b[0m | \u001b[0m 6.006   \u001b[0m | \u001b[0m 1.927   \u001b[0m | \u001b[0m 476.8   \u001b[0m | \u001b[0m 59.14   \u001b[0m | \u001b[0m 47.11   \u001b[0m | \u001b[0m 0.6499  \u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m-5.28e+06\u001b[0m | \u001b[0m 0.5441  \u001b[0m | \u001b[0m 0.6661  \u001b[0m | \u001b[0m 0.783   \u001b[0m | \u001b[0m 0.2454  \u001b[0m | \u001b[0m 0.2117  \u001b[0m | \u001b[0m 9.819   \u001b[0m | \u001b[0m 6.781   \u001b[0m | \u001b[0m 4.262   \u001b[0m | \u001b[0m 138.2   \u001b[0m | \u001b[0m 25.09   \u001b[0m | \u001b[0m 81.11   \u001b[0m | \u001b[0m 0.8534  \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m-5.207e+0\u001b[0m | \u001b[0m 0.6373  \u001b[0m | \u001b[0m 0.5471  \u001b[0m | \u001b[0m 0.984   \u001b[0m | \u001b[0m 0.3728  \u001b[0m | \u001b[0m 0.2596  \u001b[0m | \u001b[0m 1.759   \u001b[0m | \u001b[0m 7.068   \u001b[0m | \u001b[0m 5.765   \u001b[0m | \u001b[0m 858.5   \u001b[0m | \u001b[0m 16.83   \u001b[0m | \u001b[0m 93.27   \u001b[0m | \u001b[0m 0.9327  \u001b[0m |\n",
      "| \u001b[95m 13      \u001b[0m | \u001b[95m-1.541e+0\u001b[0m | \u001b[95m 0.8708  \u001b[0m | \u001b[95m 0.506   \u001b[0m | \u001b[95m 0.966   \u001b[0m | \u001b[95m 0.8969  \u001b[0m | \u001b[95m 0.159   \u001b[0m | \u001b[95m 0.3267  \u001b[0m | \u001b[95m 5.764   \u001b[0m | \u001b[95m 3.897   \u001b[0m | \u001b[95m 328.9   \u001b[0m | \u001b[95m 33.29   \u001b[0m | \u001b[95m 15.83   \u001b[0m | \u001b[95m 0.7789  \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m-5.408e+0\u001b[0m | \u001b[0m 0.7416  \u001b[0m | \u001b[0m 0.6178  \u001b[0m | \u001b[0m 0.9234  \u001b[0m | \u001b[0m 0.4121  \u001b[0m | \u001b[0m 0.1972  \u001b[0m | \u001b[0m 3.485   \u001b[0m | \u001b[0m 9.593   \u001b[0m | \u001b[0m 2.534   \u001b[0m | \u001b[0m 122.3   \u001b[0m | \u001b[0m 26.69   \u001b[0m | \u001b[0m 28.61   \u001b[0m | \u001b[0m 0.6902  \u001b[0m |\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m-5.09e+06\u001b[0m | \u001b[0m 0.8839  \u001b[0m | \u001b[0m 0.6197  \u001b[0m | \u001b[0m 0.8403  \u001b[0m | \u001b[0m 0.7373  \u001b[0m | \u001b[0m 0.1395  \u001b[0m | \u001b[0m 9.246   \u001b[0m | \u001b[0m 5.555   \u001b[0m | \u001b[0m 6.901   \u001b[0m | \u001b[0m 585.0   \u001b[0m | \u001b[0m 11.88   \u001b[0m | \u001b[0m 29.26   \u001b[0m | \u001b[0m 0.76    \u001b[0m |\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m-5.164e+0\u001b[0m | \u001b[0m 0.5781  \u001b[0m | \u001b[0m 0.9239  \u001b[0m | \u001b[0m 0.7311  \u001b[0m | \u001b[0m 0.8392  \u001b[0m | \u001b[0m 0.2236  \u001b[0m | \u001b[0m 4.248   \u001b[0m | \u001b[0m 7.164   \u001b[0m | \u001b[0m 7.223   \u001b[0m | \u001b[0m 612.5   \u001b[0m | \u001b[0m 83.28   \u001b[0m | \u001b[0m 89.77   \u001b[0m | \u001b[0m 0.5027  \u001b[0m |\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m-5.419e+0\u001b[0m | \u001b[0m 0.657   \u001b[0m | \u001b[0m 0.5288  \u001b[0m | \u001b[0m 0.5905  \u001b[0m | \u001b[0m 0.8366  \u001b[0m | \u001b[0m 0.04852 \u001b[0m | \u001b[0m 4.828   \u001b[0m | \u001b[0m 6.617   \u001b[0m | \u001b[0m 5.763   \u001b[0m | \u001b[0m 234.7   \u001b[0m | \u001b[0m 32.27   \u001b[0m | \u001b[0m 21.55   \u001b[0m | \u001b[0m 0.9705  \u001b[0m |\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m-5.444e+0\u001b[0m | \u001b[0m 0.7088  \u001b[0m | \u001b[0m 0.9463  \u001b[0m | \u001b[0m 0.6093  \u001b[0m | \u001b[0m 0.02088 \u001b[0m | \u001b[0m 0.1205  \u001b[0m | \u001b[0m 2.838   \u001b[0m | \u001b[0m 6.805   \u001b[0m | \u001b[0m 8.687   \u001b[0m | \u001b[0m 65.67   \u001b[0m | \u001b[0m 14.35   \u001b[0m | \u001b[0m 52.7    \u001b[0m | \u001b[0m 0.8862  \u001b[0m |\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m-5.208e+0\u001b[0m | \u001b[0m 0.9643  \u001b[0m | \u001b[0m 0.87    \u001b[0m | \u001b[0m 0.6656  \u001b[0m | \u001b[0m 0.02176 \u001b[0m | \u001b[0m 0.07554 \u001b[0m | \u001b[0m 8.701   \u001b[0m | \u001b[0m 8.136   \u001b[0m | \u001b[0m 2.285   \u001b[0m | \u001b[0m 646.9   \u001b[0m | \u001b[0m 51.24   \u001b[0m | \u001b[0m 91.01   \u001b[0m | \u001b[0m 0.9463  \u001b[0m |\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m-5.241e+0\u001b[0m | \u001b[0m 0.9138  \u001b[0m | \u001b[0m 0.7998  \u001b[0m | \u001b[0m 0.5992  \u001b[0m | \u001b[0m 0.1759  \u001b[0m | \u001b[0m 0.2822  \u001b[0m | \u001b[0m 4.09    \u001b[0m | \u001b[0m 6.89    \u001b[0m | \u001b[0m 6.762   \u001b[0m | \u001b[0m 331.5   \u001b[0m | \u001b[0m 29.64   \u001b[0m | \u001b[0m 16.78   \u001b[0m | \u001b[0m 0.5573  \u001b[0m |\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m-5.358e+0\u001b[0m | \u001b[0m 0.8686  \u001b[0m | \u001b[0m 0.9418  \u001b[0m | \u001b[0m 0.951   \u001b[0m | \u001b[0m 0.4164  \u001b[0m | \u001b[0m 0.03344 \u001b[0m | \u001b[0m 7.973   \u001b[0m | \u001b[0m 8.498   \u001b[0m | \u001b[0m 3.277   \u001b[0m | \u001b[0m 560.8   \u001b[0m | \u001b[0m 85.45   \u001b[0m | \u001b[0m 38.48   \u001b[0m | \u001b[0m 0.5498  \u001b[0m |\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m-5.185e+0\u001b[0m | \u001b[0m 0.637   \u001b[0m | \u001b[0m 0.7876  \u001b[0m | \u001b[0m 0.6555  \u001b[0m | \u001b[0m 0.3076  \u001b[0m | \u001b[0m 0.1786  \u001b[0m | \u001b[0m 3.137   \u001b[0m | \u001b[0m 8.719   \u001b[0m | \u001b[0m 5.657   \u001b[0m | \u001b[0m 938.5   \u001b[0m | \u001b[0m 2.933   \u001b[0m | \u001b[0m 47.78   \u001b[0m | \u001b[0m 0.801   \u001b[0m |\n",
      "| \u001b[0m 23      \u001b[0m | \u001b[0m-5.058e+0\u001b[0m | \u001b[0m 0.7118  \u001b[0m | \u001b[0m 0.9607  \u001b[0m | \u001b[0m 0.8121  \u001b[0m | \u001b[0m 0.7286  \u001b[0m | \u001b[0m 0.2284  \u001b[0m | \u001b[0m 4.836   \u001b[0m | \u001b[0m 9.142   \u001b[0m | \u001b[0m 1.585   \u001b[0m | \u001b[0m 720.0   \u001b[0m | \u001b[0m 57.6    \u001b[0m | \u001b[0m 15.41   \u001b[0m | \u001b[0m 0.5151  \u001b[0m |\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m-5.244e+0\u001b[0m | \u001b[0m 0.57    \u001b[0m | \u001b[0m 0.7953  \u001b[0m | \u001b[0m 0.561   \u001b[0m | \u001b[0m 0.1079  \u001b[0m | \u001b[0m 0.1138  \u001b[0m | \u001b[0m 8.192   \u001b[0m | \u001b[0m 8.11    \u001b[0m | \u001b[0m 6.509   \u001b[0m | \u001b[0m 406.2   \u001b[0m | \u001b[0m 93.4    \u001b[0m | \u001b[0m 74.24   \u001b[0m | \u001b[0m 0.9538  \u001b[0m |\n",
      "| \u001b[0m 25      \u001b[0m | \u001b[0m-5.028e+0\u001b[0m | \u001b[0m 0.9654  \u001b[0m | \u001b[0m 0.5604  \u001b[0m | \u001b[0m 0.7796  \u001b[0m | \u001b[0m 0.9627  \u001b[0m | \u001b[0m 0.2452  \u001b[0m | \u001b[0m 3.528   \u001b[0m | \u001b[0m 8.674   \u001b[0m | \u001b[0m 2.099   \u001b[0m | \u001b[0m 931.5   \u001b[0m | \u001b[0m 92.87   \u001b[0m | \u001b[0m 19.01   \u001b[0m | \u001b[0m 0.7408  \u001b[0m |\n",
      "| \u001b[0m 26      \u001b[0m | \u001b[0m-5.288e+0\u001b[0m | \u001b[0m 0.8568  \u001b[0m | \u001b[0m 0.7364  \u001b[0m | \u001b[0m 0.5014  \u001b[0m | \u001b[0m 0.1874  \u001b[0m | \u001b[0m 0.163   \u001b[0m | \u001b[0m 5.529   \u001b[0m | \u001b[0m 5.967   \u001b[0m | \u001b[0m 7.564   \u001b[0m | \u001b[0m 286.3   \u001b[0m | \u001b[0m 91.28   \u001b[0m | \u001b[0m 48.8    \u001b[0m | \u001b[0m 0.8318  \u001b[0m |\n",
      "| \u001b[0m 27      \u001b[0m | \u001b[0m-5.093e+0\u001b[0m | \u001b[0m 0.6062  \u001b[0m | \u001b[0m 0.5788  \u001b[0m | \u001b[0m 0.7365  \u001b[0m | \u001b[0m 0.5989  \u001b[0m | \u001b[0m 0.2752  \u001b[0m | \u001b[0m 4.925   \u001b[0m | \u001b[0m 9.153   \u001b[0m | \u001b[0m 1.896   \u001b[0m | \u001b[0m 538.5   \u001b[0m | \u001b[0m 48.69   \u001b[0m | \u001b[0m 37.99   \u001b[0m | \u001b[0m 0.6103  \u001b[0m |\n",
      "| \u001b[0m 28      \u001b[0m | \u001b[0m-5.164e+0\u001b[0m | \u001b[0m 0.76    \u001b[0m | \u001b[0m 0.5517  \u001b[0m | \u001b[0m 0.5389  \u001b[0m | \u001b[0m 0.8436  \u001b[0m | \u001b[0m 0.2948  \u001b[0m | \u001b[0m 1.784   \u001b[0m | \u001b[0m 5.303   \u001b[0m | \u001b[0m 2.368   \u001b[0m | \u001b[0m 970.3   \u001b[0m | \u001b[0m 77.52   \u001b[0m | \u001b[0m 67.98   \u001b[0m | \u001b[0m 0.855   \u001b[0m |\n",
      "| \u001b[0m 29      \u001b[0m | \u001b[0m-5.097e+0\u001b[0m | \u001b[0m 0.9839  \u001b[0m | \u001b[0m 0.7158  \u001b[0m | \u001b[0m 0.6892  \u001b[0m | \u001b[0m 0.04748 \u001b[0m | \u001b[0m 0.09934 \u001b[0m | \u001b[0m 7.781   \u001b[0m | \u001b[0m 9.682   \u001b[0m | \u001b[0m 3.61    \u001b[0m | \u001b[0m 902.9   \u001b[0m | \u001b[0m 72.65   \u001b[0m | \u001b[0m 58.48   \u001b[0m | \u001b[0m 0.5118  \u001b[0m |\n",
      "| \u001b[0m 30      \u001b[0m | \u001b[0m-5.039e+0\u001b[0m | \u001b[0m 0.9897  \u001b[0m | \u001b[0m 0.6123  \u001b[0m | \u001b[0m 0.782   \u001b[0m | \u001b[0m 0.5161  \u001b[0m | \u001b[0m 0.1517  \u001b[0m | \u001b[0m 6.848   \u001b[0m | \u001b[0m 9.202   \u001b[0m | \u001b[0m 5.67    \u001b[0m | \u001b[0m 829.1   \u001b[0m | \u001b[0m 39.01   \u001b[0m | \u001b[0m 6.906   \u001b[0m | \u001b[0m 0.5719  \u001b[0m |\n",
      "| \u001b[0m 31      \u001b[0m | \u001b[0m-5.353e+0\u001b[0m | \u001b[0m 0.9784  \u001b[0m | \u001b[0m 0.8823  \u001b[0m | \u001b[0m 0.5185  \u001b[0m | \u001b[0m 0.6     \u001b[0m | \u001b[0m 0.05369 \u001b[0m | \u001b[0m 4.046   \u001b[0m | \u001b[0m 8.229   \u001b[0m | \u001b[0m 6.12    \u001b[0m | \u001b[0m 739.7   \u001b[0m | \u001b[0m 18.6    \u001b[0m | \u001b[0m 50.37   \u001b[0m | \u001b[0m 0.5415  \u001b[0m |\n",
      "| \u001b[0m 32      \u001b[0m | \u001b[0m-5.232e+0\u001b[0m | \u001b[0m 0.7035  \u001b[0m | \u001b[0m 0.8704  \u001b[0m | \u001b[0m 0.9543  \u001b[0m | \u001b[0m 0.08201 \u001b[0m | \u001b[0m 0.2662  \u001b[0m | \u001b[0m 2.56    \u001b[0m | \u001b[0m 9.865   \u001b[0m | \u001b[0m 9.546   \u001b[0m | \u001b[0m 482.2   \u001b[0m | \u001b[0m 3.236   \u001b[0m | \u001b[0m 8.767   \u001b[0m | \u001b[0m 0.8971  \u001b[0m |\n",
      "| \u001b[0m 33      \u001b[0m | \u001b[0m-5.161e+0\u001b[0m | \u001b[0m 0.8609  \u001b[0m | \u001b[0m 0.8877  \u001b[0m | \u001b[0m 0.8728  \u001b[0m | \u001b[0m 0.002598\u001b[0m | \u001b[0m 0.2934  \u001b[0m | \u001b[0m 5.163   \u001b[0m | \u001b[0m 9.058   \u001b[0m | \u001b[0m 5.074   \u001b[0m | \u001b[0m 373.5   \u001b[0m | \u001b[0m 39.85   \u001b[0m | \u001b[0m 7.648   \u001b[0m | \u001b[0m 0.5945  \u001b[0m |\n",
      "| \u001b[0m 34      \u001b[0m | \u001b[0m-5.257e+0\u001b[0m | \u001b[0m 0.7367  \u001b[0m | \u001b[0m 0.8157  \u001b[0m | \u001b[0m 0.8865  \u001b[0m | \u001b[0m 0.3736  \u001b[0m | \u001b[0m 0.1236  \u001b[0m | \u001b[0m 2.615   \u001b[0m | \u001b[0m 5.062   \u001b[0m | \u001b[0m 9.356   \u001b[0m | \u001b[0m 908.1   \u001b[0m | \u001b[0m 18.14   \u001b[0m | \u001b[0m 87.07   \u001b[0m | \u001b[0m 0.7246  \u001b[0m |\n",
      "| \u001b[0m 35      \u001b[0m | \u001b[0m-5.001e+0\u001b[0m | \u001b[0m 0.7867  \u001b[0m | \u001b[0m 0.68    \u001b[0m | \u001b[0m 0.8148  \u001b[0m | \u001b[0m 0.8844  \u001b[0m | \u001b[0m 0.2126  \u001b[0m | \u001b[0m 8.501   \u001b[0m | \u001b[0m 5.795   \u001b[0m | \u001b[0m 8.703   \u001b[0m | \u001b[0m 520.7   \u001b[0m | \u001b[0m 29.68   \u001b[0m | \u001b[0m 2.695   \u001b[0m | \u001b[0m 0.8348  \u001b[0m |\n",
      "| \u001b[0m 36      \u001b[0m | \u001b[0m-5.362e+0\u001b[0m | \u001b[0m 0.9321  \u001b[0m | \u001b[0m 0.7474  \u001b[0m | \u001b[0m 0.8679  \u001b[0m | \u001b[0m 0.5255  \u001b[0m | \u001b[0m 0.231   \u001b[0m | \u001b[0m 0.9356  \u001b[0m | \u001b[0m 5.475   \u001b[0m | \u001b[0m 9.537   \u001b[0m | \u001b[0m 618.4   \u001b[0m | \u001b[0m 56.8    \u001b[0m | \u001b[0m 71.66   \u001b[0m | \u001b[0m 0.7758  \u001b[0m |\n",
      "| \u001b[0m 37      \u001b[0m | \u001b[0m-5.353e+0\u001b[0m | \u001b[0m 0.6878  \u001b[0m | \u001b[0m 0.8862  \u001b[0m | \u001b[0m 0.7301  \u001b[0m | \u001b[0m 0.9271  \u001b[0m | \u001b[0m 0.1265  \u001b[0m | \u001b[0m 5.747   \u001b[0m | \u001b[0m 7.197   \u001b[0m | \u001b[0m 5.896   \u001b[0m | \u001b[0m 209.5   \u001b[0m | \u001b[0m 43.63   \u001b[0m | \u001b[0m 36.2    \u001b[0m | \u001b[0m 0.5122  \u001b[0m |\n",
      "| \u001b[0m 38      \u001b[0m | \u001b[0m-1.574e+0\u001b[0m | \u001b[0m 0.7262  \u001b[0m | \u001b[0m 0.9321  \u001b[0m | \u001b[0m 0.8869  \u001b[0m | \u001b[0m 0.2002  \u001b[0m | \u001b[0m 0.1108  \u001b[0m | \u001b[0m 0.1061  \u001b[0m | \u001b[0m 7.57    \u001b[0m | \u001b[0m 5.685   \u001b[0m | \u001b[0m 783.4   \u001b[0m | \u001b[0m 10.18   \u001b[0m | \u001b[0m 72.41   \u001b[0m | \u001b[0m 0.7634  \u001b[0m |\n",
      "| \u001b[0m 39      \u001b[0m | \u001b[0m-5.242e+0\u001b[0m | \u001b[0m 0.6819  \u001b[0m | \u001b[0m 0.9063  \u001b[0m | \u001b[0m 0.6265  \u001b[0m | \u001b[0m 0.9493  \u001b[0m | \u001b[0m 0.2272  \u001b[0m | \u001b[0m 6.178   \u001b[0m | \u001b[0m 8.803   \u001b[0m | \u001b[0m 5.32    \u001b[0m | \u001b[0m 271.1   \u001b[0m | \u001b[0m 79.5    \u001b[0m | \u001b[0m 15.52   \u001b[0m | \u001b[0m 0.7679  \u001b[0m |\n",
      "| \u001b[0m 40      \u001b[0m | \u001b[0m-5.415e+0\u001b[0m | \u001b[0m 0.5655  \u001b[0m | \u001b[0m 0.9827  \u001b[0m | \u001b[0m 0.9859  \u001b[0m | \u001b[0m 0.8933  \u001b[0m | \u001b[0m 0.04067 \u001b[0m | \u001b[0m 8.258   \u001b[0m | \u001b[0m 5.11    \u001b[0m | \u001b[0m 5.427   \u001b[0m | \u001b[0m 191.5   \u001b[0m | \u001b[0m 59.17   \u001b[0m | \u001b[0m 95.98   \u001b[0m | \u001b[0m 0.5611  \u001b[0m |\n",
      "| \u001b[0m 41      \u001b[0m | \u001b[0m-5.22e+06\u001b[0m | \u001b[0m 0.6299  \u001b[0m | \u001b[0m 0.5607  \u001b[0m | \u001b[0m 0.9648  \u001b[0m | \u001b[0m 0.3941  \u001b[0m | \u001b[0m 0.2403  \u001b[0m | \u001b[0m 6.367   \u001b[0m | \u001b[0m 9.115   \u001b[0m | \u001b[0m 7.49    \u001b[0m | \u001b[0m 286.6   \u001b[0m | \u001b[0m 92.84   \u001b[0m | \u001b[0m 94.8    \u001b[0m | \u001b[0m 0.8974  \u001b[0m |\n",
      "| \u001b[0m 42      \u001b[0m | \u001b[0m-4.997e+0\u001b[0m | \u001b[0m 0.6263  \u001b[0m | \u001b[0m 0.8693  \u001b[0m | \u001b[0m 0.7094  \u001b[0m | \u001b[0m 0.9668  \u001b[0m | \u001b[0m 0.2291  \u001b[0m | \u001b[0m 5.808   \u001b[0m | \u001b[0m 5.779   \u001b[0m | \u001b[0m 3.468   \u001b[0m | \u001b[0m 740.6   \u001b[0m | \u001b[0m 69.8    \u001b[0m | \u001b[0m 36.35   \u001b[0m | \u001b[0m 0.648   \u001b[0m |\n",
      "| \u001b[0m 43      \u001b[0m | \u001b[0m-5.144e+0\u001b[0m | \u001b[0m 0.5241  \u001b[0m | \u001b[0m 0.9085  \u001b[0m | \u001b[0m 0.8543  \u001b[0m | \u001b[0m 0.6481  \u001b[0m | \u001b[0m 0.1963  \u001b[0m | \u001b[0m 5.64    \u001b[0m | \u001b[0m 8.699   \u001b[0m | \u001b[0m 5.627   \u001b[0m | \u001b[0m 500.5   \u001b[0m | \u001b[0m 34.0    \u001b[0m | \u001b[0m 42.7    \u001b[0m | \u001b[0m 0.7603  \u001b[0m |\n",
      "| \u001b[0m 44      \u001b[0m | \u001b[0m-4.621e+0\u001b[0m | \u001b[0m 0.9641  \u001b[0m | \u001b[0m 0.8985  \u001b[0m | \u001b[0m 0.5617  \u001b[0m | \u001b[0m 0.06873 \u001b[0m | \u001b[0m 0.296   \u001b[0m | \u001b[0m 9.214   \u001b[0m | \u001b[0m 8.038   \u001b[0m | \u001b[0m 6.607   \u001b[0m | \u001b[0m 972.4   \u001b[0m | \u001b[0m 82.19   \u001b[0m | \u001b[0m 57.26   \u001b[0m | \u001b[0m 0.6076  \u001b[0m |\n",
      "| \u001b[0m 45      \u001b[0m | \u001b[0m-5.428e+0\u001b[0m | \u001b[0m 0.5495  \u001b[0m | \u001b[0m 0.5398  \u001b[0m | \u001b[0m 0.9359  \u001b[0m | \u001b[0m 0.1313  \u001b[0m | \u001b[0m 0.07979 \u001b[0m | \u001b[0m 3.719   \u001b[0m | \u001b[0m 7.312   \u001b[0m | \u001b[0m 9.856   \u001b[0m | \u001b[0m 140.1   \u001b[0m | \u001b[0m 67.6    \u001b[0m | \u001b[0m 2.259   \u001b[0m | \u001b[0m 0.9755  \u001b[0m |\n",
      "| \u001b[0m 46      \u001b[0m | \u001b[0m-4.889e+0\u001b[0m | \u001b[0m 0.8846  \u001b[0m | \u001b[0m 0.8083  \u001b[0m | \u001b[0m 0.5969  \u001b[0m | \u001b[0m 0.6645  \u001b[0m | \u001b[0m 0.259   \u001b[0m | \u001b[0m 6.392   \u001b[0m | \u001b[0m 5.467   \u001b[0m | \u001b[0m 5.106   \u001b[0m | \u001b[0m 906.2   \u001b[0m | \u001b[0m 28.3    \u001b[0m | \u001b[0m 63.35   \u001b[0m | \u001b[0m 0.778   \u001b[0m |\n",
      "| \u001b[0m 47      \u001b[0m | \u001b[0m-5.454e+0\u001b[0m | \u001b[0m 0.8727  \u001b[0m | \u001b[0m 0.7254  \u001b[0m | \u001b[0m 0.8507  \u001b[0m | \u001b[0m 0.1452  \u001b[0m | \u001b[0m 0.02926 \u001b[0m | \u001b[0m 3.466   \u001b[0m | \u001b[0m 7.199   \u001b[0m | \u001b[0m 1.715   \u001b[0m | \u001b[0m 129.4   \u001b[0m | \u001b[0m 94.67   \u001b[0m | \u001b[0m 93.0    \u001b[0m | \u001b[0m 0.9805  \u001b[0m |\n",
      "| \u001b[0m 48      \u001b[0m | \u001b[0m-5.453e+0\u001b[0m | \u001b[0m 0.7957  \u001b[0m | \u001b[0m 0.8554  \u001b[0m | \u001b[0m 0.7213  \u001b[0m | \u001b[0m 0.7007  \u001b[0m | \u001b[0m 0.0115  \u001b[0m | \u001b[0m 1.81    \u001b[0m | \u001b[0m 5.829   \u001b[0m | \u001b[0m 6.852   \u001b[0m | \u001b[0m 569.3   \u001b[0m | \u001b[0m 94.52   \u001b[0m | \u001b[0m 83.19   \u001b[0m | \u001b[0m 0.6728  \u001b[0m |\n",
      "| \u001b[0m 49      \u001b[0m | \u001b[0m-5.383e+0\u001b[0m | \u001b[0m 0.5016  \u001b[0m | \u001b[0m 0.5319  \u001b[0m | \u001b[0m 0.951   \u001b[0m | \u001b[0m 0.8874  \u001b[0m | \u001b[0m 0.02814 \u001b[0m | \u001b[0m 3.909   \u001b[0m | \u001b[0m 8.382   \u001b[0m | \u001b[0m 9.37    \u001b[0m | \u001b[0m 966.8   \u001b[0m | \u001b[0m 75.27   \u001b[0m | \u001b[0m 38.66   \u001b[0m | \u001b[0m 0.6708  \u001b[0m |\n",
      "| \u001b[0m 50      \u001b[0m | \u001b[0m-5.412e+0\u001b[0m | \u001b[0m 0.5343  \u001b[0m | \u001b[0m 0.691   \u001b[0m | \u001b[0m 0.9247  \u001b[0m | \u001b[0m 0.9406  \u001b[0m | \u001b[0m 0.08512 \u001b[0m | \u001b[0m 0.6733  \u001b[0m | \u001b[0m 9.371   \u001b[0m | \u001b[0m 5.527   \u001b[0m | \u001b[0m 786.4   \u001b[0m | \u001b[0m 14.55   \u001b[0m | \u001b[0m 72.96   \u001b[0m | \u001b[0m 0.5633  \u001b[0m |\n",
      "| \u001b[0m 51      \u001b[0m | \u001b[0m-4.885e+0\u001b[0m | \u001b[0m 0.607   \u001b[0m | \u001b[0m 0.7355  \u001b[0m | \u001b[0m 0.5422  \u001b[0m | \u001b[0m 0.02778 \u001b[0m | \u001b[0m 0.2544  \u001b[0m | \u001b[0m 6.966   \u001b[0m | \u001b[0m 7.025   \u001b[0m | \u001b[0m 3.755   \u001b[0m | \u001b[0m 815.0   \u001b[0m | \u001b[0m 6.004   \u001b[0m | \u001b[0m 93.48   \u001b[0m | \u001b[0m 0.8217  \u001b[0m |\n",
      "| \u001b[0m 52      \u001b[0m | \u001b[0m-5.435e+0\u001b[0m | \u001b[0m 0.672   \u001b[0m | \u001b[0m 0.8131  \u001b[0m | \u001b[0m 0.7589  \u001b[0m | \u001b[0m 0.3034  \u001b[0m | \u001b[0m 0.22    \u001b[0m | \u001b[0m 2.685   \u001b[0m | \u001b[0m 5.226   \u001b[0m | \u001b[0m 3.062   \u001b[0m | \u001b[0m 54.0    \u001b[0m | \u001b[0m 58.08   \u001b[0m | \u001b[0m 38.13   \u001b[0m | \u001b[0m 0.5842  \u001b[0m |\n",
      "| \u001b[0m 53      \u001b[0m | \u001b[0m-5.216e+0\u001b[0m | \u001b[0m 0.5599  \u001b[0m | \u001b[0m 0.9787  \u001b[0m | \u001b[0m 0.7358  \u001b[0m | \u001b[0m 0.2758  \u001b[0m | \u001b[0m 0.07942 \u001b[0m | \u001b[0m 6.576   \u001b[0m | \u001b[0m 9.989   \u001b[0m | \u001b[0m 8.799   \u001b[0m | \u001b[0m 762.0   \u001b[0m | \u001b[0m 28.4    \u001b[0m | \u001b[0m 73.19   \u001b[0m | \u001b[0m 0.7061  \u001b[0m |\n",
      "| \u001b[0m 54      \u001b[0m | \u001b[0m-5.159e+0\u001b[0m | \u001b[0m 0.7935  \u001b[0m | \u001b[0m 0.6729  \u001b[0m | \u001b[0m 0.5088  \u001b[0m | \u001b[0m 0.9176  \u001b[0m | \u001b[0m 0.2006  \u001b[0m | \u001b[0m 3.633   \u001b[0m | \u001b[0m 9.411   \u001b[0m | \u001b[0m 2.767   \u001b[0m | \u001b[0m 716.2   \u001b[0m | \u001b[0m 17.35   \u001b[0m | \u001b[0m 26.47   \u001b[0m | \u001b[0m 0.6369  \u001b[0m |\n",
      "| \u001b[0m 55      \u001b[0m | \u001b[0m-5.226e+0\u001b[0m | \u001b[0m 0.7696  \u001b[0m | \u001b[0m 0.5743  \u001b[0m | \u001b[0m 0.6454  \u001b[0m | \u001b[0m 0.3153  \u001b[0m | \u001b[0m 0.08924 \u001b[0m | \u001b[0m 9.963   \u001b[0m | \u001b[0m 5.423   \u001b[0m | \u001b[0m 6.66    \u001b[0m | \u001b[0m 457.8   \u001b[0m | \u001b[0m 97.81   \u001b[0m | \u001b[0m 12.76   \u001b[0m | \u001b[0m 0.9517  \u001b[0m |\n",
      "=========================================================================================================================================================================\n",
      "-----------------------------------------------------\n",
      "Final Results\n",
      "XGBOOST: -5445421.619478\n",
      "XGBOOST: -5134862.129115\n",
      "XGBOOST: -5180109.931803\n",
      "XGBOOST: -5273164.075382\n",
      "XGBOOST: -5371970.918531\n",
      "XGBOOST: -5399020.425110\n",
      "XGBOOST: -5055037.230537\n",
      "XGBOOST: -1611147.031367\n",
      "XGBOOST: -5121659.299545\n",
      "XGBOOST: -5387533.812013\n",
      "XGBOOST: -5280390.649896\n",
      "XGBOOST: -5207235.743572\n",
      "XGBOOST: -1540911.754923\n",
      "XGBOOST: -5408161.010799\n",
      "XGBOOST: -5089865.162887\n",
      "XGBOOST: -5163584.015512\n",
      "XGBOOST: -5419039.741040\n",
      "XGBOOST: -5444287.708479\n",
      "XGBOOST: -5208395.238507\n",
      "XGBOOST: -5240872.985352\n",
      "XGBOOST: -5357524.570962\n",
      "XGBOOST: -5184758.004362\n",
      "XGBOOST: -5058430.052762\n",
      "XGBOOST: -5244304.292509\n",
      "XGBOOST: -5028341.893011\n",
      "XGBOOST: -5287604.765111\n",
      "XGBOOST: -5092818.906111\n",
      "XGBOOST: -5163860.958110\n",
      "XGBOOST: -5096615.881434\n",
      "XGBOOST: -5038535.056502\n",
      "XGBOOST: -5352692.497997\n",
      "XGBOOST: -5232257.925079\n",
      "XGBOOST: -5160724.832742\n",
      "XGBOOST: -5256595.855201\n",
      "XGBOOST: -5001144.417419\n",
      "XGBOOST: -5362143.558799\n",
      "XGBOOST: -5352563.695691\n",
      "XGBOOST: -1573523.382718\n",
      "XGBOOST: -5242133.457202\n",
      "XGBOOST: -5415292.877215\n",
      "XGBOOST: -5220041.764477\n",
      "XGBOOST: -4996669.855302\n",
      "XGBOOST: -5144385.068706\n",
      "XGBOOST: -4620554.858261\n",
      "XGBOOST: -5428176.976183\n",
      "XGBOOST: -4889370.720146\n",
      "XGBOOST: -5454405.055385\n",
      "XGBOOST: -5452961.049718\n",
      "XGBOOST: -5383480.134929\n",
      "XGBOOST: -5411892.353717\n",
      "XGBOOST: -4884884.041434\n",
      "XGBOOST: -5435075.018840\n",
      "XGBOOST: -5215966.876474\n",
      "XGBOOST: -5159174.498285\n",
      "XGBOOST: -5225683.809848\n",
      "*****************************************************\n",
      "best params ==> {'target': -1540911.7549229823, 'params': {'colsample_bylevel': 0.870775219488481, 'colsample_bynode': 0.5059601984870159, 'colsample_bytree': 0.9659705635318063, 'gamma': 0.8968843524222182, 'learning_rate': 0.15898134999123564, 'max_delta_step': 0.3267179403769993, 'max_depth': 5.764005925896605, 'min_child_weight': 3.89657602973688, 'n_estimators': 328.9092284043236, 'reg_alpha': 33.2939268552398, 'reg_lambda': 15.830287605007618, 'subsample': 0.7788690674320253}}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from bayes_opt import BayesianOptimization\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import mean_squared_log_error, mean_squared_error\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "\n",
    "def xgboostcv(max_depth,\n",
    "              learning_rate,\n",
    "              n_estimators,\n",
    "              gamma,\n",
    "              min_child_weight,\n",
    "              max_delta_step,\n",
    "              subsample,\n",
    "              reg_lambda,\n",
    "              reg_alpha,\n",
    "              colsample_bytree,\n",
    "              colsample_bylevel,\n",
    "              colsample_bynode,\n",
    "              verbosity=0,\n",
    "              nthread=-1):\n",
    "    \n",
    "    model=xgb.XGBRegressor(max_depth=int(round(max_depth)),\n",
    "                           learning_rate=learning_rate,\n",
    "                           n_estimators=int(round(n_estimators)),\n",
    "                           gamma=gamma,\n",
    "                           min_child_weight=min_child_weight,\n",
    "                           max_delta_step=int(round(max_delta_step)),\n",
    "                           subsample=subsample,\n",
    "                           reg_lambda=reg_lambda,\n",
    "                           reg_alpha=reg_alpha,\n",
    "                           colsample_bytree=colsample_bytree,\n",
    "                           colsample_bylevel=colsample_bylevel,\n",
    "                           colsample_bynode=colsample_bynode,\n",
    "                           verbosity=verbosity,\n",
    "                           nthread=nthread,\n",
    "                           tree_method='gpu_hist')\n",
    "    \n",
    "    cv = ShuffleSplit(n_splits=5, test_size=0.2, random_state=0)\n",
    "    result = cross_val_score(model,train_X,train_y,scoring=\"neg_mean_squared_error\",cv=cv).mean()\n",
    "    \n",
    "    return result\n",
    "\n",
    "\n",
    "# Load data set and target values\n",
    "xgboostBO = BayesianOptimization(f = xgboostcv,\n",
    "                                 pbounds = {'max_depth': (5, 10),\n",
    "                                          'learning_rate': (0.01, 0.3),\n",
    "                                          'n_estimators': (50, 1000),\n",
    "                                          'gamma': (0, 0.99),\n",
    "                                          'min_child_weight': (1, 10),\n",
    "                                          'max_delta_step': (0, 10),\n",
    "                                          'subsample': (0.5, 0.99),\n",
    "                                          'reg_lambda': (1, 100),\n",
    "                                          'reg_alpha': (1, 100),\n",
    "                                          'colsample_bytree' :(0.5, 0.99),\n",
    "                                          'colsample_bylevel' :(0.5, 0.99),\n",
    "                                          'colsample_bynode' :(0.5, 0.99)\n",
    "                                          },\n",
    "                                random_state=1)\n",
    "\n",
    "xgboostBO.maximize(init_points=5, n_iter=50)\n",
    "print('-'*53)\n",
    "\n",
    "print('Final Results')\n",
    "for res in xgboostBO.res:\n",
    "    print('XGBOOST: %f' % res['target'])\n",
    "\n",
    "print('*'*53)\n",
    "print('best params ==>', xgboostBO.max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "execution": {
     "iopub.execute_input": "2020-11-07T04:40:07.323403Z",
     "iopub.status.busy": "2020-11-07T04:40:07.322583Z",
     "iopub.status.idle": "2020-11-07T04:40:07.326744Z",
     "shell.execute_reply": "2020-11-07T04:40:07.325763Z"
    },
    "papermill": {
     "duration": 0.043692,
     "end_time": "2020-11-07T04:40:07.326851",
     "exception": false,
     "start_time": "2020-11-07T04:40:07.283159",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "param_dict = xgboostBO.max['params'] \n",
    "param_dict['tree_method']='gpu_hist'\n",
    "\n",
    "# we need to round some float values to integer since these parameters works only in integer\n",
    "param_dict['max_depth']=int(round(param_dict['max_depth']))\n",
    "param_dict['n_estimators']=int(round(param_dict['n_estimators']))\n",
    "param_dict['max_delta_step']=int(round(param_dict['max_delta_step']))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-07T04:40:07.386704Z",
     "iopub.status.busy": "2020-11-07T04:40:07.386063Z",
     "iopub.status.idle": "2020-11-07T04:40:07.392857Z",
     "shell.execute_reply": "2020-11-07T04:40:07.393368Z"
    },
    "papermill": {
     "duration": 0.039238,
     "end_time": "2020-11-07T04:40:07.393484",
     "exception": false,
     "start_time": "2020-11-07T04:40:07.354246",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=None, booster=None, colsample_bylevel=0.870775219488481,\n",
       "             colsample_bynode=0.5059601984870159,\n",
       "             colsample_bytree=0.9659705635318063, gamma=0.8968843524222182,\n",
       "             gpu_id=None, importance_type='gain', interaction_constraints=None,\n",
       "             learning_rate=0.15898134999123564, max_delta_step=0, max_depth=6,\n",
       "             min_child_weight=3.89657602973688, missing=nan,\n",
       "             monotone_constraints=None, n_estimators=329, n_jobs=None,\n",
       "             num_parallel_tree=None, random_state=None,\n",
       "             reg_alpha=33.2939268552398, reg_lambda=15.830287605007618,\n",
       "             scale_pos_weight=None, subsample=0.7788690674320253,\n",
       "             tree_method='gpu_hist', validate_parameters=None, verbosity=None)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_xgb_model = xgb.XGBRegressor()\n",
    "best_xgb_model.set_params(**param_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-07T04:40:07.454098Z",
     "iopub.status.busy": "2020-11-07T04:40:07.452939Z",
     "iopub.status.idle": "2020-11-07T04:40:25.076543Z",
     "shell.execute_reply": "2020-11-07T04:40:25.075228Z"
    },
    "papermill": {
     "duration": 17.656316,
     "end_time": "2020-11-07T04:40:25.076663",
     "exception": false,
     "start_time": "2020-11-07T04:40:07.420347",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree',\n",
       "             colsample_bylevel=0.870775219488481,\n",
       "             colsample_bynode=0.5059601984870159,\n",
       "             colsample_bytree=0.9659705635318063, gamma=0.8968843524222182,\n",
       "             gpu_id=0, importance_type='gain', interaction_constraints='',\n",
       "             learning_rate=0.15898134999123564, max_delta_step=0, max_depth=6,\n",
       "             min_child_weight=3.89657602973688, missing=nan,\n",
       "             monotone_constraints='(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)',\n",
       "             n_estimators=329, n_jobs=0, num_parallel_tree=1, random_state=0,\n",
       "             reg_alpha=33.2939268552398, reg_lambda=15.830287605007618,\n",
       "             scale_pos_weight=1, subsample=0.7788690674320253,\n",
       "             tree_method='gpu_hist', validate_parameters=1, verbosity=None)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_xgb_model.fit(train_X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-07T04:40:25.142076Z",
     "iopub.status.busy": "2020-11-07T04:40:25.140699Z",
     "iopub.status.idle": "2020-11-07T04:40:27.591992Z",
     "shell.execute_reply": "2020-11-07T04:40:27.590918Z"
    },
    "papermill": {
     "duration": 2.487011,
     "end_time": "2020-11-07T04:40:27.592124",
     "exception": false,
     "start_time": "2020-11-07T04:40:25.105113",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y_pred = best_xgb_model.predict(normalized_test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-07T04:40:27.763884Z",
     "iopub.status.busy": "2020-11-07T04:40:27.762885Z",
     "iopub.status.idle": "2020-11-07T04:40:27.955592Z",
     "shell.execute_reply": "2020-11-07T04:40:27.954971Z"
    },
    "papermill": {
     "duration": 0.332944,
     "end_time": "2020-11-07T04:40:27.955719",
     "exception": false,
     "start_time": "2020-11-07T04:40:27.622775",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# prediction results\n",
    "pred_df=pd.DataFrame(zip(df_test1['sector_id'], y_pred), columns=['sector_id', 'predictions'])\n",
    "submission_dff=pred_df.groupby('sector_id').sum()\n",
    "ss=submission_dff.reset_index()\n",
    "\n",
    "ss=ss.rename(columns={\"sector_id\": \"Sektor_Id\", \"predictions\": \"Expected\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-07T04:40:28.020423Z",
     "iopub.status.busy": "2020-11-07T04:40:28.019658Z",
     "iopub.status.idle": "2020-11-07T04:40:28.292216Z",
     "shell.execute_reply": "2020-11-07T04:40:28.291570Z"
    },
    "papermill": {
     "duration": 0.307054,
     "end_time": "2020-11-07T04:40:28.292381",
     "exception": false,
     "start_time": "2020-11-07T04:40:27.985327",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "ss.to_csv('out_bayesian.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 11046.320383,
   "end_time": "2020-11-07T04:40:29.349074",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-11-07T01:36:23.028691",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}