submission-writing-quality-kaylaipp.ipynb

{"cells":[{"source":"<a href=\"https://www.kaggle.com/code/kaylaippp/copy-of-0-63-submission-writing-quality-kaylaipp?scriptVersionId=191978384\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"markdown","id":"a1489a5b","metadata":{"papermill":{"duration":0.006517,"end_time":"2024-08-10T20:04:50.331422","exception":false,"start_time":"2024-08-10T20:04:50.324905","status":"completed"},"tags":[]},"source":["<font color=blue><h1><b>Imports</b></h1></font>"]},{"cell_type":"code","execution_count":1,"id":"9d5ab7eb","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:04:50.346634Z","iopub.status.busy":"2024-08-10T20:04:50.346235Z","iopub.status.idle":"2024-08-10T20:04:55.335377Z","shell.execute_reply":"2024-08-10T20:04:55.334208Z"},"papermill":{"duration":4.999163,"end_time":"2024-08-10T20:04:55.338276","exception":false,"start_time":"2024-08-10T20:04:50.339113","status":"completed"},"tags":[]},"outputs":[],"source":["from sklearn.model_selection import train_test_split\n","from sklearn.preprocessing import StandardScaler\n","from sklearn.compose import ColumnTransformer\n","from sklearn.pipeline import Pipeline, make_pipeline\n","from sklearn.preprocessing import MinMaxScaler, StandardScaler, OneHotEncoder,power_transform\n","from sklearn.metrics import mean_squared_error\n","from xgboost import XGBRegressor\n","from sklearn.ensemble import RandomForestRegressor\n","from lightgbm import LGBMRegressor\n","from sklearn.model_selection import cross_val_score, GridSearchCV\n","from sklearn.svm import SVR\n","from sklearn.ensemble import GradientBoostingRegressor, StackingRegressor\n","from sklearn.linear_model import Ridge, Lasso, LinearRegression\n","from sklearn.neural_network import MLPRegressor\n","from catboost import CatBoostRegressor\n","\n","\n","import math\n","import scipy.stats as stats\n","import numpy as np\n","import pandas as pd\n","import time\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","from tqdm import tqdm\n","from collections import Counter\n","import re\n","from scipy.stats import entropy"]},{"cell_type":"code","execution_count":2,"id":"075cf374","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:04:55.352353Z","iopub.status.busy":"2024-08-10T20:04:55.351942Z","iopub.status.idle":"2024-08-10T20:05:12.737658Z","shell.execute_reply":"2024-08-10T20:05:12.736732Z"},"papermill":{"duration":17.395756,"end_time":"2024-08-10T20:05:12.74032","exception":false,"start_time":"2024-08-10T20:04:55.344564","status":"completed"},"tags":[]},"outputs":[],"source":["INPUT_DIR = '/kaggle/input/linking-writing-processes-to-writing-quality'\n","trainLogs_df = pd.read_csv(f'{INPUT_DIR}/train_logs.csv')\n","trainScores_df = pd.read_csv(f'{INPUT_DIR}/train_scores.csv')\n","validationLogs_df = pd.read_csv(f'{INPUT_DIR}/test_logs.csv')"]},{"cell_type":"markdown","id":"d1a4c78a","metadata":{"papermill":{"duration":0.005649,"end_time":"2024-08-10T20:05:12.75226","exception":false,"start_time":"2024-08-10T20:05:12.746611","status":"completed"},"tags":[]},"source":["<font color=blue><h1><b>Feature Engineering</b></h1></font>"]},{"cell_type":"code","execution_count":3,"id":"6a90cfcb","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:05:12.766364Z","iopub.status.busy":"2024-08-10T20:05:12.76593Z","iopub.status.idle":"2024-08-10T20:05:12.810583Z","shell.execute_reply":"2024-08-10T20:05:12.809222Z"},"papermill":{"duration":0.055,"end_time":"2024-08-10T20:05:12.813248","exception":false,"start_time":"2024-08-10T20:05:12.758248","status":"completed"},"tags":[]},"outputs":[],"source":["# helper fucntion for calculating verbosity feature\n","def calculate_entropy(x):\n","    return entropy(x / np.sum(x))\n","\n","# helper fucntion for calculating verbosity features\n","def calculate_slope(x):\n","    return np.polyfit(np.arange(len(x)), x, 1)[0]\n","\n","# helper fucntion for calculating verbosity feature\n","def calculate_uniformity(x):\n","    uniform_distribution = np.ones_like(x) / len(x)\n","    return entropy(x, uniform_distribution)\n","\n","# https://link.springer.com/article/10.1007/s40593-021-00268-w\n","def create_verbosity_features(df: pd.DataFrame): \n","    new_df = df.copy(deep=True)\n","    grouped_df = new_df.groupby('id')\n","    \n","    # Total Number of Keystrokes\n","    new_df['total_keystrokes'] = grouped_df['action_time'].transform('sum')\n","    \n","    # Total number of words\n","    new_df['total_words'] = grouped_df['word_count'].transform('sum')\n","    \n","    # SD number of keystrokes per 30 s\n","    window_size = 30  # seconds\n","    new_df['keystrokes_per_window'] = grouped_df['action_time'].transform('sum')\n","    new_df['sd_keystrokes_per_30s'] = grouped_df['keystrokes_per_window'].transform('std')\n","    \n","    # Slope of the number of keystrokes per 30 s.\n","    new_df['slope_keystrokes_per_30s'] = grouped_df['action_time'].transform(calculate_slope)\n","    \n","    # Entropy of the number of keystrokes per 30 s\n","    new_df['entropy_keystrokes_per_30s'] = grouped_df['action_time'].transform(calculate_entropy)\n","    \n","    # Uniformity of the number of keystrokes per 30 s.\n","    new_df['uniformity_keystrokes_per_30s'] = grouped_df['action_time'].transform(calculate_uniformity)\n","    \n","    # Local extreme number of keystrokes per 30 s\n","    new_df['keystrokes_diff_sign'] = np.sign(grouped_df['action_time'].diff().diff())\n","    new_df['local_extreme_count'] = grouped_df['keystrokes_diff_sign'].apply(lambda x: np.sum(x != 0))\n","    \n","    # Mean and SD distance 30 s windows of more than one keystroke\n","    new_df['keystrokes_per_window'] = grouped_df['action_time'].transform('sum')\n","    windows_with_more_than_one_keystroke = new_df[new_df['keystrokes_per_window'] > 1].groupby('id')\n","    new_df['mean_distance_between_windows'] = windows_with_more_than_one_keystroke['action_time'].diff().mean()\n","    new_df['sd_distance_between_windows'] = windows_with_more_than_one_keystroke['action_time'].diff().std()\n","    \n","    # Drop helper cols that we dont need anymore\n","    new_df = new_df.drop(['keystrokes_per_window', 'keystrokes_diff_sign'], axis=1)\n","    \n","    return new_df\n","\n","\n","# Basic data preparation\n","def create_features(df: pd.DataFrame):\n","    # Stolen shamlessely from \"Writing Quality LOFO Feature Importance\" here:\n","    # https://www.kaggle.com/code/aerdem4/writing-quality-lofo-feature-importance\n","    new_df = df\n","    new_df['wait_time'] = new_df['down_time'] - new_df.groupby('id')['up_time'].shift()\n","    new_df['activity'] = new_df['activity'].apply(lambda x: 'Move' if x.startswith('Move') else x)\n","    mask = new_df['wait_time'] < 0\n","    new_df.loc[mask, 'wait_time'] = 0\n","\n","\n","    temp_df = new_df[new_df['activity'] != 'Remove/Cut'].groupby('id').agg({'text_change': list}).reset_index()\n","    temp_df['text_change'] = temp_df['text_change'].apply(lambda x: ''.join(x)).apply(lambda x: re.findall(r'q+', x))\n","    temp_df['input_word_length_mean'] = temp_df['text_change'].apply(lambda x: np.mean([len(i) for i in x] if len(x) > 0 else 0))\n","    temp_df['input_word_length_max'] = temp_df['text_change'].apply(lambda x: np.max([len(i) for i in x] if len(x) > 0 else 0))\n","    temp_df['input_word_length_std'] = temp_df['text_change'].apply(lambda x: np.std([len(i) for i in x] if len(x) > 0 else 0))\n","    temp_df.drop(['text_change'], axis=1, inplace=True)\n","\n","\n","    # One hot encoding of activity\n","    #activity_ohe_df = pd.get_dummies(df['activity'])\n","    #activity_cols = list(activity_ohe_df.columns)\n","\n","    # Remerge ohe columns with rest of df\n","    #df = pd.concat([df, activity_ohe_df], axis=1)\n","    # Create some aggregate statistic features\n","    agg = { \"event_id\" : [\"last\"],\n","        \"wait_time\": [\"mean\", \"max\", \"sum\"],\n","        \"word_count\": [\"last\"],\n","        \"up_time\": [ \"max\"],\n","        \"down_time\": [\"min\"],\n","        \"valid_sentences\" : [\"sum\"],\n","        \"valid_paras\" : [\"sum\"],\n","         \"Nav_action_time\" : [\"sum\"],\n","         \"Wasted_action_time\" : [\"sum\"],\n","         \"Wasted_Action_Events\" : [\"sum\"]\n","        }\n","\n","    shiftedCols = ['cursor_position', 'word_count']\n","    gaps = [10,100, 200]\n","\n","    getTimeGapData(new_df, shiftedCols, gaps)\n","    getActionGapData (new_df, gaps)\n","    for gap in gaps:\n","        agg[\"action_time_shift_\" + str(gap)] = [\"mean\", \"max\", \"std\"]\n","        for col in shiftedCols:\n","              agg[str(col) + \"_shift_\" + str(gap)] = [\"mean\", \"max\", \"std\"]\n","\n","    pbar = agg\n","    colNames = []\n","    for item in pbar:\n","        colname, methods = item[0], item[1]\n","        for method in methods:\n","            colNames.append(colname + \"_\" + method)\n","    new_df = new_df.groupby(['id']).agg(agg).reset_index()\n","    new_df.columns = [str(col[0] + \"_\" + col[1]) for col in new_df.columns] # some colum name cleaning\n","\n","\n","  # Taken and modified from https://www.kaggle.com/code/abhranta/lgbm-finetuning-with-optuna\n","    new_df['input_word_length_mean'] = temp_df['input_word_length_mean']\n","    new_df['input_word_length_max'] = temp_df['input_word_length_max']\n","    new_df['input_word_length_std'] = temp_df['input_word_length_std']\n","\n","    # Our own development\n","    new_df['total_time'] = new_df['up_time_max'] - new_df['down_time_min']\n","    new_df.drop(['up_time_max', 'down_time_min'], axis=1, inplace=True)\n","\n","\n","    new_df[\"valid_sentences_sum\"] = new_df[\"valid_sentences_sum\"] + 1\n","    new_df[\"valid_paras_sum\"] = new_df[\"valid_paras_sum\"] + 1\n","    new_df[\"Nav_time_ratio\"] = new_df [\"Nav_action_time_sum\"] / new_df['total_time']\n","    new_df['Wasted_time_ratio'] = new_df['Wasted_action_time_sum'] / new_df['total_time']\n","    new_df['Wasted_actions_per_word'] = new_df['Wasted_Action_Events_sum'] / new_df['word_count_last']\n","    new_df['Idle_time_ratio'] = new_df['wait_time_sum'] / new_df['total_time']\n","\n","    new_df['Word_rate'] = new_df['word_count_last'] / new_df['total_time']\n","    new_df['Word_event_rate'] = new_df['word_count_last'] / new_df['event_id_last']\n","    new_df['event_rate'] = new_df['event_id_last'] / new_df['total_time']\n","    new_df['Sentences_per_para'] = new_df['valid_sentences_sum']/ new_df['valid_paras_sum']\n","    new_df['words_per_sentences'] = new_df['word_count_last']/ new_df['valid_sentences_sum']\n","    new_df.drop(['Nav_action_time_sum', 'Wasted_action_time_sum', 'Wasted_Action_Events_sum', 'event_id_last' ], axis=1, inplace=True)\n","    return new_df\n","\n","\n","def findTotalValidSentences(df:pd.DataFrame):\n","    df['valid_sentences'] = (df ['down_event'] == '.')\n","    return df\n","\n","def findTotalParagraphs(df:pd.DataFrame):\n","    df['Shifted .'] = df.groupby('id')['down_event'].shift(1)\n","    df['valid_paras'] = ((df ['down_event'] == 'Enter') & (df['Shifted .'] != 'Enter'))\n","    df.drop('Shifted .' , axis = 1, inplace = True)\n","    return df\n","\n","def findNavActions(df:pd.DataFrame):\n","    mask = ((df['up_event'] == 'ArrowDown') | (df['up_event'] == 'ArrowUp') | (df['up_event'] == 'ArrowLeft') | (df['up_event'] == 'ArrowRight'))\n","    print(mask.value_counts())  \n","    df['Nav_action_time'] = 0\n","    print(df[mask]['action_time'].sum())\n","    df.loc[mask, 'Nav_action_time'] =df.loc[mask, 'action_time']\n","    print(df['Nav_action_time'].sum())\n","    return df\n","\n","def totalWastedTime(df:pd.DataFrame):\n","    mask = df['activity'] == 'Remove/Cut'\n","    df['Wasted_action_time'] = 0\n","    df['Wasted_Action_Events'] = mask\n","    df.loc[mask, 'Wasted_action_time'] =df.loc[mask, 'action_time']\n","    print(df['Wasted_action_time'].sum())\n","    return df\n","\n","\n","def getTimeGapData(df:pd.DataFrame, colList, gapList):\n","    for col in colList:\n","        for gap in gapList:\n","            df[\"temp\"] = df.groupby('id')[col].shift(gap)\n","            df[str(col) + \"_shift_\" + str(gap)] = np.abs(df[col] - df[\"temp\"])\n","            df.drop(columns=[\"temp\"], inplace=True)\n","            df[str(col) + \"_shift_\" + str(gap)].fillna(0, inplace=True)\n","        \n","        \n","def getActionGapData (df:pd.DataFrame, gapList):\n","    for gap in gapList:\n","        df[\"up_time_shift_\" + str(gap)] = df.groupby('id')['up_time'].shift(gap)\n","        df[\"action_time_shift_\" + str(gap)] = df['down_time'] - df[\"up_time_shift_\" + str(gap)]\n","        df[\"action_time_shift_\" + str(gap)].fillna(0, inplace = True)\n","        df.drop(columns = [\"up_time_shift_\" + str(gap)], inplace = True)"]},{"cell_type":"markdown","id":"3694e7e9","metadata":{"papermill":{"duration":0.005722,"end_time":"2024-08-10T20:05:12.824965","exception":false,"start_time":"2024-08-10T20:05:12.819243","status":"completed"},"tags":[]},"source":["<font color=blue><h1><b>Apply feature engineering</b></h1></font>"]},{"cell_type":"code","execution_count":4,"id":"d9aff6b6","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:05:12.838553Z","iopub.status.busy":"2024-08-10T20:05:12.838135Z","iopub.status.idle":"2024-08-10T20:06:24.662907Z","shell.execute_reply":"2024-08-10T20:06:24.661725Z"},"papermill":{"duration":71.840419,"end_time":"2024-08-10T20:06:24.671332","exception":false,"start_time":"2024-08-10T20:05:12.830913","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["up_event\n","False    8162280\n","True      243618\n","Name: count, dtype: int64\n","15337923\n","15337923\n","72942446\n","applied features on training logs\n","up_event\n","False    6\n","Name: count, dtype: int64\n","0\n","0\n","0\n","applied features on validation logs\n"]}],"source":["trainlogs_2 = trainLogs_df.copy(deep = True)\n","trainlogs_2 = findTotalValidSentences(trainlogs_2)\n","trainlogs_2 = findTotalParagraphs(trainlogs_2)\n","trainlogs_2 = findNavActions(trainlogs_2)\n","trainlogs_2 = totalWastedTime (trainlogs_2)\n","trainlogs_2 = create_verbosity_features(trainlogs_2)\n","trainlogs_2 = create_features (trainlogs_2)\n","print('applied features on training logs')\n","\n","\n","validationlogs_2 = validationLogs_df.copy(deep = True)\n","validationlogs_2 = findTotalValidSentences(validationlogs_2)\n","validationlogs_2 = findTotalParagraphs(validationlogs_2)\n","validationlogs_2 = findNavActions(validationlogs_2)\n","validationlogs_2 = totalWastedTime (validationlogs_2)\n","validationlogs_2 = create_verbosity_features(validationlogs_2)\n","validationlogs_2 = create_features (validationlogs_2)\n","validationlogs_2.fillna(0, inplace = True)\n","print('applied features on validation logs')"]},{"cell_type":"code","execution_count":5,"id":"6be83a82","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:06:24.686098Z","iopub.status.busy":"2024-08-10T20:06:24.685627Z","iopub.status.idle":"2024-08-10T20:06:24.708155Z","shell.execute_reply":"2024-08-10T20:06:24.706782Z"},"papermill":{"duration":0.033243,"end_time":"2024-08-10T20:06:24.710942","exception":false,"start_time":"2024-08-10T20:06:24.677699","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Index(['wait_time_mean', 'wait_time_max', 'wait_time_sum', 'word_count_last',\n","       'valid_sentences_sum', 'valid_paras_sum', 'action_time_shift_10_mean',\n","       'action_time_shift_10_max', 'action_time_shift_10_std',\n","       'cursor_position_shift_10_mean', 'cursor_position_shift_10_max',\n","       'cursor_position_shift_10_std', 'word_count_shift_10_mean',\n","       'word_count_shift_10_max', 'word_count_shift_10_std',\n","       'action_time_shift_100_mean', 'action_time_shift_100_max',\n","       'action_time_shift_100_std', 'cursor_position_shift_100_mean',\n","       'cursor_position_shift_100_max', 'cursor_position_shift_100_std',\n","       'word_count_shift_100_mean', 'word_count_shift_100_max',\n","       'word_count_shift_100_std', 'action_time_shift_200_mean',\n","       'action_time_shift_200_max', 'action_time_shift_200_std',\n","       'cursor_position_shift_200_mean', 'cursor_position_shift_200_max',\n","       'cursor_position_shift_200_std', 'word_count_shift_200_mean',\n","       'word_count_shift_200_max', 'word_count_shift_200_std',\n","       'input_word_length_mean', 'input_word_length_max',\n","       'input_word_length_std', 'total_time', 'Nav_time_ratio',\n","       'Wasted_time_ratio', 'Wasted_actions_per_word', 'Idle_time_ratio',\n","       'Word_rate', 'Word_event_rate', 'event_rate', 'Sentences_per_para',\n","       'words_per_sentences'],\n","      dtype='object')\n"]}],"source":["# drop id columns\n","\n","trainlogs_2.rename(columns={\"id_\":\"id\"}, inplace=True)\n","trainlogs_2 = trainlogs_2.merge(trainScores_df, on = 'id')\n","trainlogs_2.drop(columns='id', axis=1, inplace=True)\n","\n","validationlogs_2.rename(columns={\"id_\":\"id\"}, inplace=True)\n","validation_ids = validationlogs_2['id']\n","validationlogs_2.drop(columns='id', axis=1, inplace=True)\n","features = trainlogs_2.columns[:-1]\n","print(features)"]},{"cell_type":"code","execution_count":6,"id":"00e29d5b","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:06:24.726292Z","iopub.status.busy":"2024-08-10T20:06:24.725865Z","iopub.status.idle":"2024-08-10T20:06:25.009892Z","shell.execute_reply":"2024-08-10T20:06:25.00869Z"},"papermill":{"duration":0.29489,"end_time":"2024-08-10T20:06:25.012698","exception":false,"start_time":"2024-08-10T20:06:24.717808","status":"completed"},"tags":[]},"outputs":[],"source":["from sklearn.preprocessing import PowerTransformer\n","\n","pt = PowerTransformer(method='yeo-johnson', standardize=True)\n","trainlogs_2[features] = pt.fit_transform(trainlogs_2[features])\n","validationlogs_2[features] = pt.transform(validationlogs_2[features])"]},{"cell_type":"code","execution_count":7,"id":"97dccbb2","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:06:25.027121Z","iopub.status.busy":"2024-08-10T20:06:25.026718Z","iopub.status.idle":"2024-08-10T20:06:25.036272Z","shell.execute_reply":"2024-08-10T20:06:25.035172Z"},"papermill":{"duration":0.02047,"end_time":"2024-08-10T20:06:25.039517","exception":false,"start_time":"2024-08-10T20:06:25.019047","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Index(['wait_time_mean', 'wait_time_max', 'wait_time_sum', 'word_count_last',\n","       'valid_sentences_sum', 'valid_paras_sum', 'action_time_shift_10_mean',\n","       'action_time_shift_10_max', 'action_time_shift_10_std',\n","       'cursor_position_shift_10_mean', 'cursor_position_shift_10_max',\n","       'cursor_position_shift_10_std', 'word_count_shift_10_mean',\n","       'word_count_shift_10_max', 'word_count_shift_10_std',\n","       'action_time_shift_100_mean', 'action_time_shift_100_max',\n","       'action_time_shift_100_std', 'cursor_position_shift_100_mean',\n","       'cursor_position_shift_100_max', 'cursor_position_shift_100_std',\n","       'word_count_shift_100_mean', 'word_count_shift_100_max',\n","       'word_count_shift_100_std', 'action_time_shift_200_mean',\n","       'action_time_shift_200_max', 'action_time_shift_200_std',\n","       'cursor_position_shift_200_mean', 'cursor_position_shift_200_max',\n","       'cursor_position_shift_200_std', 'word_count_shift_200_mean',\n","       'word_count_shift_200_max', 'word_count_shift_200_std',\n","       'input_word_length_mean', 'input_word_length_max',\n","       'input_word_length_std', 'total_time', 'Nav_time_ratio',\n","       'Wasted_time_ratio', 'Wasted_actions_per_word', 'Idle_time_ratio',\n","       'Word_rate', 'Word_event_rate', 'event_rate', 'Sentences_per_para',\n","       'words_per_sentences'],\n","      dtype='object')\n"]}],"source":["trainScore = trainlogs_2['score']\n","trainlogs_2.drop('score', axis = 1, inplace = True)\n","print(trainlogs_2.columns)"]},{"cell_type":"code","execution_count":8,"id":"ada65e59","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:06:25.054479Z","iopub.status.busy":"2024-08-10T20:06:25.054063Z","iopub.status.idle":"2024-08-10T20:06:32.232562Z","shell.execute_reply":"2024-08-10T20:06:32.229618Z"},"papermill":{"duration":7.191534,"end_time":"2024-08-10T20:06:32.237697","exception":false,"start_time":"2024-08-10T20:06:25.046163","status":"completed"},"tags":[]},"outputs":[],"source":["from sklearn.feature_selection import SequentialFeatureSelector\n","from sklearn.linear_model import LinearRegression\n","\n","lr = LinearRegression()\n","sfs = SequentialFeatureSelector(lr, direction=\"backward\", scoring = 'neg_mean_absolute_error', tol = 0.001, n_features_to_select='auto')\n","sfs.fit(trainlogs_2, trainScore)\n","sfs.get_support()\n","sfs.get_feature_names_out(features)\n","\n","trainlogs_2 = sfs.transform(trainlogs_2)\n","validationlogs_2 = sfs.transform(validationlogs_2)"]},{"cell_type":"markdown","id":"863a4b47","metadata":{"papermill":{"duration":0.03746,"end_time":"2024-08-10T20:06:32.301085","exception":false,"start_time":"2024-08-10T20:06:32.263625","status":"completed"},"tags":[]},"source":["### MSE scores"]},{"cell_type":"code","execution_count":9,"id":"6c46074e","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:06:32.319758Z","iopub.status.busy":"2024-08-10T20:06:32.318776Z","iopub.status.idle":"2024-08-10T20:06:32.325902Z","shell.execute_reply":"2024-08-10T20:06:32.324876Z"},"papermill":{"duration":0.017316,"end_time":"2024-08-10T20:06:32.328303","exception":false,"start_time":"2024-08-10T20:06:32.310987","status":"completed"},"tags":[]},"outputs":[],"source":["def print_mse(model):\n","    ## output training and validation MSE (on training subset)\n","    x_train, x_test, y_train, y_test = train_test_split(trainlogs_2, trainScore, test_size=0.20, random_state=43)\n","    y_pred = model.predict(x_test)\n","    y_pred = [5 if x > 5 else x for x in y_pred]\n","    print('training MSE: ', math.sqrt(mean_squared_error(y_train, model.predict(x_train))))\n","    print('validation MSE: ', math.sqrt(mean_squared_error(y_test, y_pred)))"]},{"cell_type":"markdown","id":"6547642c","metadata":{"papermill":{"duration":0.00632,"end_time":"2024-08-10T20:06:32.341059","exception":false,"start_time":"2024-08-10T20:06:32.334739","status":"completed"},"tags":[]},"source":["<font color=blue><h1><b> Stacked model </b></h1></font>\n","\n","- stacking 3 models, CatBoostRegressor, RandomForestRegressor and a NN (MLPRegressor) and using linear regression as the final estimator "]},{"cell_type":"code","execution_count":10,"id":"5c94debb","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:06:32.355829Z","iopub.status.busy":"2024-08-10T20:06:32.355421Z","iopub.status.idle":"2024-08-10T20:13:14.080908Z","shell.execute_reply":"2024-08-10T20:13:14.079721Z"},"papermill":{"duration":401.745064,"end_time":"2024-08-10T20:13:14.092819","exception":false,"start_time":"2024-08-10T20:06:32.347755","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["training MSE:  0.45585646280758346\n","validation MSE:  0.4395598073972519\n"]}],"source":["nn = MLPRegressor(hidden_layer_sizes=(100,), max_iter=5000, activation='logistic')\n","cat_boost_model = CatBoostRegressor(\n","    iterations=300,         \n","    learning_rate=0.05,     \n","    l2_leaf_reg= 1,\n","    depth=5,                \n","    loss_function='RMSE',\n","    random_seed=42,\n","    verbose=0\n",")\n","random_forest_model = RandomForestRegressor(random_state=42, bootstrap=True, max_depth=20, min_samples_leaf=4, min_samples_split=2, n_estimators=300)\n","stacked_model = StackingRegressor(\n","    estimators=[\n","        ('mlpr', make_pipeline(StandardScaler(), nn)),\n","        ('rf', make_pipeline(StandardScaler(), random_forest_model)),\n","        ('cbr', make_pipeline(StandardScaler(), cat_boost_model)),\n","    ],\n","    final_estimator=LinearRegression(),\n","    cv=7\n",")\n","stacked_model.fit(trainlogs_2, trainScore)\n","y_pred = stacked_model.predict(validationlogs_2)\n","submission_df = pd.DataFrame({\n","    'id': validation_ids,\n","    'score': y_pred\n","})\n","\n","## output training and validation MSE (on training subset)\n","print_mse(stacked_model)"]},{"cell_type":"code","execution_count":11,"id":"39a1af8a","metadata":{"execution":{"iopub.execute_input":"2024-08-10T20:13:14.111844Z","iopub.status.busy":"2024-08-10T20:13:14.111391Z","iopub.status.idle":"2024-08-10T20:13:14.122158Z","shell.execute_reply":"2024-08-10T20:13:14.120806Z"},"papermill":{"duration":0.022789,"end_time":"2024-08-10T20:13:14.125266","exception":false,"start_time":"2024-08-10T20:13:14.102477","status":"completed"},"tags":[]},"outputs":[],"source":["submission_df.to_csv('submission.csv', index=False)"]}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"databundleVersionId":6678907,"sourceId":59291,"sourceType":"competition"}],"dockerImageVersionId":30558,"isGpuEnabled":false,"isInternetEnabled":false,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"papermill":{"default_parameters":{},"duration":509.099933,"end_time":"2024-08-10T20:13:15.159533","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-08-10T20:04:46.0596","version":"2.4.0"}},"nbformat":4,"nbformat_minor":5}