{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"f77f454a-0014-4fa5-a086-ad9a957cc0ad","cell_type":"markdown","source":"# **imports**","metadata":{}},{"id":"1668e21f-e4f8-46cb-9ba5-899423f92987","cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport category_encoders as ce\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom tensorflow.keras import regularizers\nfrom tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, Callback\nfrom keras import regularizers\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:36:56.789757Z","iopub.execute_input":"2024-12-18T14:36:56.790242Z","iopub.status.idle":"2024-12-18T14:36:56.80131Z","shell.execute_reply.started":"2024-12-18T14:36:56.790208Z","shell.execute_reply":"2024-12-18T14:36:56.799914Z"}},"outputs":[],"execution_count":null},{"id":"952cb64c-db9f-469a-85f1-446c696463d0","cell_type":"markdown","source":"# **functions**","metadata":{}},{"id":"7c9b6b1e-286b-43b7-9f66-4361c9724612","cell_type":"code","source":"\ndef split_numerical_categorical(df):\n    \"\"\"\n    Splits the columns of a DataFrame into numerical and categorical features.\n\n    Parameters:\n    df (pandas.DataFrame): The DataFrame to split.\n\n    Returns:\n    tuple: A tuple containing two lists - numerical columns and categorical columns.\n    \"\"\"\n    numerical_cols = df.select_dtypes(include=['number']).columns.tolist()\n    categorical_cols = df.select_dtypes(exclude=['number']).columns.tolist()\n    return numerical_cols, categorical_cols\n\ndef rmsle(y_true, y_pred):\n    # Ensure predictions are not below a certain threshold\n    y_pred = tf.maximum(y_pred, 20.0)\n    \n    # Compute the RMSLE\n    msle = tf.keras.losses.MeanSquaredLogarithmicError()\n    return tf.sqrt(msle(y_true, y_pred))\n\ndef create_new_features(data):\n \n    data['Income_to_Age'] = data['Annual Income'] / data['Age']\n    \n    data['Claims_Frequency'] = data['Previous Claims'] / (data['Insurance Duration'] + 1)\n    \n    data['Dependents_to_Income'] = data['Number of Dependents'] / (data['Annual Income'] + 1)\n\n    data['Claims_to_HealthScore'] = data['Previous Claims'] / (data['Health Score'] + 1)\n    \n    data['Vehicle_Age_Adjusted_Claims'] = data['Vehicle Age'] * data['Previous Claims']\n    \n    data['Normalized_Credit_Score'] = (data['Credit Score'] - data['Credit Score'].mean()) / data['Credit Score'].std()\n    \n    data['Duration_to_Age'] = data['Insurance Duration'] / (data['Age'] + 1)\n    \n    data[\"Annual_Income_Health_Score_Ratio\"] = data[\"Annual Income\"] / data[\"Health Score\"] \n    \n    data[\"Annual_Income_Health_Score\"] = data[\"Annual Income\"] * data[\"Health Score\"]\n    \n    data[\"Annual_Income_Credit_Score_Ratio\"] = data[\"Annual Income\"] / data[\"Credit Score\"]\n\n    data[\"Annual_Income_Credit_Score\"] = data[\"Annual Income\"] * data[\"Credit Score\"]\n\n    data[\"Vehicle_Age_Insurance_Duration\"] = data[\"Vehicle Age\"] / data[\"Insurance Duration\"]\n    \n    data['Income Per Dependent'] = data['Annual Income'] / (data['Number of Dependents'] + 1)\n\n    data['Health Risk Index'] = data['Health Score'] * data['Vehicle Age']\n\n    return data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:36:58.963691Z","iopub.execute_input":"2024-12-18T14:36:58.964232Z","iopub.status.idle":"2024-12-18T14:36:58.983656Z","shell.execute_reply.started":"2024-12-18T14:36:58.96418Z","shell.execute_reply":"2024-12-18T14:36:58.982557Z"}},"outputs":[],"execution_count":null},{"id":"872b68fa-3712-4895-9498-efb4d9ef22db","cell_type":"markdown","source":"# **preparing data**","metadata":{}},{"id":"8d2c228f-865c-427c-b571-4a20028a1994","cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\nfor df in [train, test]:\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\ntest_ids = test[\"id\"]\n\nnum_cols= train.select_dtypes(include='number').columns\nnum_cols = [val for val in num_cols]\ncat_cols = train.select_dtypes(exclude='number').columns\ndatetime_cols = ['Policy Start Date']\ncat_cols = [col for col in cat_cols if col not in datetime_cols]\n\nprint(\"Numerical columns:\",)\nprint(num_cols)\nprint(\"\")\nprint(\"Categorical columns excluding datetime columns:\")\nprint(cat_cols)\nprint(\"\")\nprint(\"Datetime column:\")\nprint(datetime_cols)\n\nfor df in [train, test]:\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['Year'] = df['Policy Start Date'].dt.year\n    df = df.drop(columns = 'Policy Start Date', inplace = True)\n\nprint(\"New columns created: Day, Month, Year\")\n\ntrain = create_new_features(train)\ntest = create_new_features(test)\n\n\nnum_cols= train.select_dtypes(include='number').columns\nnum_cols = [val for val in num_cols]\n\ny_ = train['Premium Amount']\nX_ = train.drop(columns = ['Premium Amount', 'id'])\ntest_ = test.drop(columns = ['id'])\n    \nencoder = ce.TargetEncoder()\n\nfor feature in cat_cols:\n    train[feature] = encoder.fit_transform(train[feature], train['Premium Amount'])\n    test[feature] = encoder.transform(test[feature])\n\nprint(train)\n\nprint(\"Categorical columns transformed with target encoder in train and test data: \")\nprint(cat_cols)\n\ncorr = train.corr()['Premium Amount'].sort_values(ascending = False)\npd.DataFrame(corr).style.background_gradient(cmap='Blues')\nabs_corr = corr.abs()\nmedian_abs_corr = abs_corr.median()\nfiltered_corr = corr[abs_corr > median_abs_corr]\npd.DataFrame(filtered_corr).style.background_gradient(cmap='Blues')\nfiltered_columns_test = filtered_corr.index.drop('Premium Amount')\nfiltered_columns_train = filtered_corr.index\n\ntrain = train[filtered_columns_train]\ntest = test[filtered_columns_test]","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:37:07.464821Z","iopub.execute_input":"2024-12-18T14:37:07.465963Z","iopub.status.idle":"2024-12-18T14:37:38.681049Z","shell.execute_reply.started":"2024-12-18T14:37:07.465909Z","shell.execute_reply":"2024-12-18T14:37:38.680091Z"}},"outputs":[],"execution_count":null},{"id":"ae3628e3-d7e2-4f4b-be4e-ffe169a1cea0","cell_type":"code","source":"pd.DataFrame(corr).style.background_gradient(cmap='Blues')","metadata":{"scrolled":true},"outputs":[],"execution_count":null},{"id":"7275d612-fb08-4797-9207-449da351f21c","cell_type":"code","source":"y = train['Premium Amount']\nX = train.drop(columns = ['Premium Amount'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:37:49.269014Z","iopub.execute_input":"2024-12-18T14:37:49.26945Z","iopub.status.idle":"2024-12-18T14:37:49.368606Z","shell.execute_reply.started":"2024-12-18T14:37:49.269404Z","shell.execute_reply":"2024-12-18T14:37:49.367257Z"}},"outputs":[],"execution_count":null},{"id":"ffc78405-8b2b-4abd-9f70-816da4f917ee","cell_type":"code","source":"scaler = StandardScaler()\nX = X.fillna(0.0)\ntest = test.fillna(0.0)\ncolumns = X.columns\nX = scaler.fit_transform(X)\nX = pd.DataFrame(X, columns=columns)\ntest = scaler.transform(test)\ntest = pd.DataFrame(test, columns=columns)\nX.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:37:50.251783Z","iopub.execute_input":"2024-12-18T14:37:50.252205Z","iopub.status.idle":"2024-12-18T14:37:50.873338Z","shell.execute_reply.started":"2024-12-18T14:37:50.252168Z","shell.execute_reply":"2024-12-18T14:37:50.872255Z"}},"outputs":[],"execution_count":null},{"id":"57490e8e-af53-43ee-ad81-3b08f828b786","cell_type":"code","source":"X_train, X_testing, y_train,  y_testing = train_test_split(X, y, test_size=0.2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:37:53.901446Z","iopub.execute_input":"2024-12-18T14:37:53.902405Z","iopub.status.idle":"2024-12-18T14:37:54.172987Z","shell.execute_reply.started":"2024-12-18T14:37:53.902365Z","shell.execute_reply":"2024-12-18T14:37:54.172044Z"}},"outputs":[],"execution_count":null},{"id":"440ba809-3655-4f89-816f-5c70815e10f7","cell_type":"code","source":"X_train.shape[1]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:37:55.253319Z","iopub.execute_input":"2024-12-18T14:37:55.253728Z","iopub.status.idle":"2024-12-18T14:37:55.260223Z","shell.execute_reply.started":"2024-12-18T14:37:55.253689Z","shell.execute_reply":"2024-12-18T14:37:55.259111Z"}},"outputs":[],"execution_count":null},{"id":"b7634b81-c3ac-4ccf-bcb1-763c77c821af","cell_type":"markdown","source":"# **training the model**","metadata":{}},{"id":"cb2191f2-1b5d-4414-9ebb-6d3ea0677434","cell_type":"code","source":"# Create a Sequential model\nmodel = Sequential()\n\n# Input layer\nmodel.add(Input(shape=(X_train.shape[1],)))  \nmodel.add(BatchNormalization())\n\n# First hidden layer with L2 regularization\nmodel.add(Dense(256, activation='relu',\n                kernel_regularizer=regularizers.l2(0.01)))  \nmodel.add(BatchNormalization()) \nmodel.add(Dropout(0.1)) \n\n# Second hidden layer with L2 regularization\nmodel.add(Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.01))) \nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.1))\n\n# Fourth hidden layer with L2 regularization\nmodel.add(Dense(64, activation='relu', kernel_regularizer=regularizers.l2(0.01)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.1))\n\n# Fourth hidden layer with L2 regularization\nmodel.add(Dense(32, activation='relu', kernel_regularizer=regularizers.l2(0.01)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.1))\n\n# Output layer\nmodel.add(Dense(1, activation='linear'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:37:56.612816Z","iopub.execute_input":"2024-12-18T14:37:56.61322Z","iopub.status.idle":"2024-12-18T14:37:56.827778Z","shell.execute_reply.started":"2024-12-18T14:37:56.613183Z","shell.execute_reply":"2024-12-18T14:37:56.826807Z"}},"outputs":[],"execution_count":null},{"id":"adac98f9-2cf5-4b98-ba09-fb91972fdd7e","cell_type":"code","source":"early_stopping = EarlyStopping(monitor='rmsle', mode='min', patience=2, restore_best_weights=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:37:59.092002Z","iopub.execute_input":"2024-12-18T14:37:59.092406Z","iopub.status.idle":"2024-12-18T14:37:59.097779Z","shell.execute_reply.started":"2024-12-18T14:37:59.092373Z","shell.execute_reply":"2024-12-18T14:37:59.096475Z"}},"outputs":[],"execution_count":null},{"id":"3927600d-4c32-4534-bef4-e5b9a50e6e6e","cell_type":"code","source":"model.compile(optimizer=Adam(learning_rate=0.01, clipvalue=1.0), loss='mse', metrics=[rmsle])\nhistory = model.fit(X_train, y_train, \n                    epochs=1, \n                    batch_size=1024, \n                    validation_split=0.1,)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:38:00.773205Z","iopub.execute_input":"2024-12-18T14:38:00.774315Z","iopub.status.idle":"2024-12-18T14:38:19.598265Z","shell.execute_reply.started":"2024-12-18T14:38:00.774272Z","shell.execute_reply":"2024-12-18T14:38:19.59737Z"}},"outputs":[],"execution_count":null},{"id":"0af2241d-1ea2-4245-b260-df9ad0e94bd9","cell_type":"code","source":"model.compile(optimizer=Adam(learning_rate=0.001, clipvalue=1.0), loss=rmsle, metrics=[rmsle])\nhistory = model.fit(X_train, y_train, \n                    epochs=3, \n                    batch_size=1024, \n                    validation_split=0.1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:38:46.189666Z","iopub.execute_input":"2024-12-18T14:38:46.190789Z","iopub.status.idle":"2024-12-18T14:39:34.316373Z","shell.execute_reply.started":"2024-12-18T14:38:46.190747Z","shell.execute_reply":"2024-12-18T14:39:34.315202Z"}},"outputs":[],"execution_count":null},{"id":"71d1f43b-fa85-44f6-9733-299b0e8b7b87","cell_type":"code","source":"model.compile(optimizer=Adam(learning_rate=0.0005), loss=rmsle, metrics=[rmsle])\nhistory = model.fit(X_train, y_train, \n                    epochs=10, \n                    batch_size=2048, \n                    validation_split=0.2,\n                    callbacks=early_stopping)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:39:34.318458Z","iopub.execute_input":"2024-12-18T14:39:34.318793Z","iopub.status.idle":"2024-12-18T14:41:14.711872Z","shell.execute_reply.started":"2024-12-18T14:39:34.318761Z","shell.execute_reply":"2024-12-18T14:41:14.710931Z"}},"outputs":[],"execution_count":null},{"id":"cc0af3d3-3410-4f0e-967f-17206474f317","cell_type":"code","source":"model.compile(optimizer=Adam(learning_rate=0.00005), loss=rmsle, metrics=[rmsle])\nhistory = model.fit(X_train, y_train, \n                    epochs=10, \n                    batch_size=4096, \n                    validation_split=0.2,\n                    callbacks=early_stopping)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:48:35.660207Z","iopub.execute_input":"2024-12-18T14:48:35.66063Z","iopub.status.idle":"2024-12-18T14:49:24.027781Z","shell.execute_reply.started":"2024-12-18T14:48:35.660595Z","shell.execute_reply":"2024-12-18T14:49:24.026839Z"}},"outputs":[],"execution_count":null},{"id":"ff169127-d781-450b-93f6-8166669ab0f6","cell_type":"code","source":"testings = model.predict(X_testing)\nerror = rmsle(y_testing, testings)\nprint(error)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:49:37.154784Z","iopub.execute_input":"2024-12-18T14:49:37.155212Z","iopub.status.idle":"2024-12-18T14:49:49.28769Z","shell.execute_reply.started":"2024-12-18T14:49:37.155172Z","shell.execute_reply":"2024-12-18T14:49:49.28662Z"}},"outputs":[],"execution_count":null},{"id":"4a5458cb-51c2-482d-8a9f-a16c5a9c04fa","cell_type":"code","source":"print(testings.shape,\n     test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:49:53.275408Z","iopub.execute_input":"2024-12-18T14:49:53.275811Z","iopub.status.idle":"2024-12-18T14:49:53.2812Z","shell.execute_reply.started":"2024-12-18T14:49:53.275774Z","shell.execute_reply":"2024-12-18T14:49:53.279883Z"}},"outputs":[],"execution_count":null},{"id":"615e2fe3-1b41-4004-af2f-a0f5e0559285","cell_type":"markdown","source":"# **final predictions**","metadata":{}},{"id":"1b974f5d-9145-4f4b-b604-1c24b0891f54","cell_type":"code","source":"test_predictions = model.predict(test)\npred = np.array(test_predictions).reshape((800000, ))\n\nsubmission = pd.DataFrame({\n    \"id\": test_ids, \n    \"Premium Amount\": pred\n})\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"File saved.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:49:53.610506Z","iopub.execute_input":"2024-12-18T14:49:53.611074Z","iopub.status.idle":"2024-12-18T14:50:36.695741Z","shell.execute_reply.started":"2024-12-18T14:49:53.610995Z","shell.execute_reply":"2024-12-18T14:50:36.694627Z"}},"outputs":[],"execution_count":null},{"id":"af6ea356-1982-4dd5-94ff-87b0e74dd233","cell_type":"code","source":"pred.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:42:23.154594Z","iopub.execute_input":"2024-12-18T14:42:23.155337Z","iopub.status.idle":"2024-12-18T14:42:23.162195Z","shell.execute_reply.started":"2024-12-18T14:42:23.155288Z","shell.execute_reply":"2024-12-18T14:42:23.161163Z"}},"outputs":[],"execution_count":null},{"id":"89395668-3241-4bac-b4fb-255d1ac98966","cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}