{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport catboost as cb\n%matplotlib inline\nimport seaborn as sns\nimport optuna\nimport xgboost as xgb\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import Ridge\n\n\n# Set plotting parameters\nplt.rcParams['figure.figsize'] = (12, 8)\nsns.set()\nsns.set(font_scale=1.5)\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\n\npd.set_option('display.max_columns', 100)\npd.set_option('expand_frame_repr', True)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:18:48.150119Z","iopub.execute_input":"2024-12-31T22:18:48.150488Z","iopub.status.idle":"2024-12-31T22:18:49.767304Z","shell.execute_reply.started":"2024-12-31T22:18:48.150446Z","shell.execute_reply":"2024-12-31T22:18:49.766209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load datasets\ntrain = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv', index_col='id')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv', index_col='id')\n\n# Store target variable and drop from train data\ntrain_target = train['Premium Amount']\ntrain.drop('Premium Amount', axis=1, inplace=True)\n\ncombined = pd.concat([train, test], keys=['train', 'test'])\nprint(f'Combined data shape: {combined.shape}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:18:49.768875Z","iopub.execute_input":"2024-12-31T22:18:49.769314Z","iopub.status.idle":"2024-12-31T22:18:58.821573Z","shell.execute_reply.started":"2024-12-31T22:18:49.769285Z","shell.execute_reply":"2024-12-31T22:18:58.820584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head(20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:18:58.823328Z","iopub.execute_input":"2024-12-31T22:18:58.824022Z","iopub.status.idle":"2024-12-31T22:18:58.855351Z","shell.execute_reply.started":"2024-12-31T22:18:58.823976Z","shell.execute_reply":"2024-12-31T22:18:58.85453Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Identifying Data Types","metadata":{}},{"cell_type":"code","source":"for col in combined.columns:\n    print(f'Column {col} type: {combined[col].dtype}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:18:58.85619Z","iopub.execute_input":"2024-12-31T22:18:58.856412Z","iopub.status.idle":"2024-12-31T22:18:58.861736Z","shell.execute_reply.started":"2024-12-31T22:18:58.85639Z","shell.execute_reply":"2024-12-31T22:18:58.860967Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Checking Missing Values","metadata":{}},{"cell_type":"code","source":"def num_missing(x):\n    return x.isnull().sum()\n\nprint(\"Missing values for each column:\")\nmissing_values = combined.apply(num_missing, axis=0).where(lambda x: x != 0).dropna()\nprint(missing_values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:18:58.864036Z","iopub.execute_input":"2024-12-31T22:18:58.864377Z","iopub.status.idle":"2024-12-31T22:18:59.713434Z","shell.execute_reply.started":"2024-12-31T22:18:58.864339Z","shell.execute_reply":"2024-12-31T22:18:59.71259Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Data cleaning","metadata":{}},{"cell_type":"code","source":"cleaned_data = combined.copy()\n\n# Imputation des valeurs manquantes pour les variables numériques\nnumeric_columns = cleaned_data.select_dtypes(include=['float64', 'int64']).columns\nfor col in numeric_columns:\n    cleaned_data[col] = cleaned_data[col].fillna(cleaned_data[col].median())\n\n# Imputation des valeurs manquantes pour les variables catégoriques\ncategorical_columns = cleaned_data.select_dtypes(include=['object']).columns\nfor col in categorical_columns:\n    cleaned_data[col] = cleaned_data[col].fillna('Unknown')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:18:59.714381Z","iopub.execute_input":"2024-12-31T22:18:59.714655Z","iopub.status.idle":"2024-12-31T22:19:02.695809Z","shell.execute_reply.started":"2024-12-31T22:18:59.714631Z","shell.execute_reply":"2024-12-31T22:19:02.694564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Date transform\nif 'Policy Start Date' in cleaned_data.columns:\n    cleaned_data['Policy Start Year'] = pd.to_datetime(cleaned_data['Policy Start Date']).dt.year\n    cleaned_data['Policy Start Month'] = pd.to_datetime(cleaned_data['Policy Start Date']).dt.month\n    cleaned_data = cleaned_data.drop(columns=['Policy Start Date'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:19:02.697197Z","iopub.execute_input":"2024-12-31T22:19:02.697621Z","iopub.status.idle":"2024-12-31T22:19:04.239937Z","shell.execute_reply.started":"2024-12-31T22:19:02.697572Z","shell.execute_reply":"2024-12-31T22:19:04.238989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Encoding categorical column\nencoded_data = pd.get_dummies(cleaned_data, drop_first=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:19:04.24101Z","iopub.execute_input":"2024-12-31T22:19:04.241296Z","iopub.status.idle":"2024-12-31T22:19:06.200051Z","shell.execute_reply.started":"2024-12-31T22:19:04.241268Z","shell.execute_reply":"2024-12-31T22:19:06.199067Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Détection et traitement des valeurs aberrantes\nfor col in numeric_columns:\n    lower_bound = cleaned_data[col].quantile(0.01)\n    upper_bound = cleaned_data[col].quantile(0.99)\n    cleaned_data[col] = cleaned_data[col].clip(lower=lower_bound, upper=upper_bound)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:19:06.201205Z","iopub.execute_input":"2024-12-31T22:19:06.201491Z","iopub.status.idle":"2024-12-31T22:19:06.739685Z","shell.execute_reply.started":"2024-12-31T22:19:06.201464Z","shell.execute_reply":"2024-12-31T22:19:06.738995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_cleaned = encoded_data.loc['train']\ntest_cleaned = encoded_data.loc['test']\n\ntrain_cleaned['Premium Amount'] = train_target\n\nprint(f\"Train data shape: {train_cleaned.shape}\")\nprint(f\"Test data shape: {test_cleaned.shape}\")\ntrain_cleaned.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:19:06.740916Z","iopub.execute_input":"2024-12-31T22:19:06.741221Z","iopub.status.idle":"2024-12-31T22:19:06.783727Z","shell.execute_reply.started":"2024-12-31T22:19:06.741186Z","shell.execute_reply":"2024-12-31T22:19:06.782838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def num_missing(x):\n    return x.isnull().sum()\n\nprint(\"Missing values for each column:\")\nmissing_values = train_cleaned.apply(num_missing, axis=0).where(lambda x: x != 0).dropna()\nprint(missing_values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:19:06.785723Z","iopub.execute_input":"2024-12-31T22:19:06.78599Z","iopub.status.idle":"2024-12-31T22:19:06.828419Z","shell.execute_reply.started":"2024-12-31T22:19:06.785963Z","shell.execute_reply":"2024-12-31T22:19:06.827653Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Model training","metadata":{}},{"cell_type":"code","source":"def rmsle(y_true, y_pred):\n    y_pred = np.maximum(y_pred, 0)\n    return np.sqrt(mean_squared_log_error(y_true, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:19:06.829483Z","iopub.execute_input":"2024-12-31T22:19:06.830095Z","iopub.status.idle":"2024-12-31T22:19:06.834548Z","shell.execute_reply.started":"2024-12-31T22:19:06.830052Z","shell.execute_reply":"2024-12-31T22:19:06.833661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train_cleaned.drop(columns=['Premium Amount'])\ny = train_cleaned['Premium Amount']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:19:06.83548Z","iopub.execute_input":"2024-12-31T22:19:06.835798Z","iopub.status.idle":"2024-12-31T22:19:06.883269Z","shell.execute_reply.started":"2024-12-31T22:19:06.835757Z","shell.execute_reply":"2024-12-31T22:19:06.882458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#------------------------------------------------------------------------------\n# 1. Hyperparameters\n#------------------------------------------------------------------------------\nxgb_params = {\n    'learning_rate': 0.013322886965047288,\n    'max_depth': 11,\n    'subsample': 0.9423226538971824,\n    'colsample_bytree': 0.9820133753290056,\n    'lambda': 5.85008073351992,\n    'alpha': 5.903813412397025,\n    'tree_method': 'hist',\n    'device': 'cuda',\n    'objective': 'reg:squarederror',\n    'eval_metric': 'rmse',   # Internal XGBoost metric\n    'seed': 42\n}\n\ndtest = xgb.DMatrix(test_cleaned, enable_categorical=True)\n\ncatboost_params = {\n    'learning_rate': 0.05,\n    'depth': 10,\n    'iterations': 1000,\n    'l2_leaf_reg': 3,\n    'loss_function': 'RMSE',\n    'eval_metric': 'RMSE',\n    'task_type': 'GPU',\n    'random_seed': 42\n}\n\n#------------------------------------------------------------------------------\n# 2. Split the Data\n#------------------------------------------------------------------------------\nX_train_meta, X_val_meta, y_train_meta, y_val_meta = train_test_split(\n    X, \n    y, \n    test_size=0.2, \n    random_state=42\n)\n\n#------------------------------------------------------------------------------\n# 3. XGBoost Training\n#------------------------------------------------------------------------------\n# Create DMatrix for XGBoost\nxgb_dtrain = xgb.DMatrix(X_train_meta, label=y_train_meta, enable_categorical=True)\nxgb_dval = xgb.DMatrix(X_val_meta, label=y_val_meta, enable_categorical=True)\n\n# Train the XGBoost model (optional: add early stopping)\nxgb_model = xgb.train(\n    params=xgb_params,\n    dtrain=xgb_dtrain,\n    num_boost_round=1000,\n    evals=[(xgb_dval, 'validation')],\n    early_stopping_rounds=50,  # Adjust if needed\n    verbose_eval=100\n)\n\n# Generate validation predictions\nxgb_val_predictions = xgb_model.predict(\n    xgb_dval, \n    iteration_range=(0, xgb_model.best_iteration + 1)  # Uses the best iteration\n)\n\n# Compute RMSLE for XGBoost on validation set\nxgb_rmsle = rmsle(y_val_meta, xgb_val_predictions)\nprint(f\"XGBoost Validation RMSLE: {xgb_rmsle:.6f}\")\n\n#------------------------------------------------------------------------------\n# 4. CatBoost Training\n#------------------------------------------------------------------------------\ncatboost_model = cb.CatBoostRegressor(**catboost_params)\ncatboost_model.fit(\n    X_train_meta, \n    y_train_meta, \n    eval_set=(X_val_meta, y_val_meta),\n    verbose=100\n)\n\n# Generate validation predictions\ncatboost_val_predictions = catboost_model.predict(X_val_meta)\n\n# Compute RMSLE for CatBoost on validation set\ncatboost_rmsle = rmsle(y_val_meta, catboost_val_predictions)\nprint(f\"CatBoost Validation RMSLE: {catboost_rmsle:.6f}\")\n\n#------------------------------------------------------------------------------\n# 5. Train Meta-Model (Stacking)\n#------------------------------------------------------------------------------\n# Combine model predictions to form meta-features\nmeta_features = np.column_stack([xgb_val_predictions, catboost_val_predictions])\n\n# Train a Ridge regressor as meta-model\nmeta_model = Ridge()\nmeta_model.fit(meta_features, y_val_meta)\n\n#------------------------------------------------------------------------------\n# 6. Final Predictions on Test Data\n#------------------------------------------------------------------------------\n# Prepare data for XGBoost \nxgb_test_predictions = xgb_model.predict(\n    dtest, \n    iteration_range=(0, xgb_model.best_iteration + 1)\n)\n\n# CatBoost test predictions\ncatboost_test_predictions = catboost_model.predict(test_cleaned)\n\n# Combine the predictions for the meta-model\ntest_meta_features = np.column_stack([xgb_test_predictions, catboost_test_predictions])\nfinal_predictions = meta_model.predict(test_meta_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:28:57.945884Z","iopub.execute_input":"2024-12-31T22:28:57.946488Z","iopub.status.idle":"2024-12-31T22:29:31.08188Z","shell.execute_reply.started":"2024-12-31T22:28:57.946451Z","shell.execute_reply":"2024-12-31T22:29:31.080625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Préparation du fichier de soumission\nsubmission = pd.DataFrame({\n    'id': test.index,\n    'Premium Amount': final_predictions\n})\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Fichier de soumission créé : 'submission.csv'\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}