{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":10306991,"sourceType":"datasetVersion","datasetId":6298404},{"sourceId":10307002,"sourceType":"datasetVersion","datasetId":6379991}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Preprocessing and Feature Eng..","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport joblib\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Data preprocessing\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.impute import SimpleImputer\n\n# Model evaluation and validation\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import mean_squared_log_error\n\n# TensorFlow/Keras\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, BatchNormalization\nfrom tensorflow.keras import regularizers\nimport keras_tuner as kt\n\n# Optuna for hyperparameter optimization\nimport optuna\n\n# Regressors\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor, Pool\nfrom sklearn.linear_model import SGDRegressor\n\n#Blending\nfrom scipy.optimize import differential_evolution","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-27T00:27:58.08125Z","iopub.execute_input":"2024-12-27T00:27:58.081657Z","iopub.status.idle":"2024-12-27T00:27:58.087308Z","shell.execute_reply.started":"2024-12-27T00:27:58.081624Z","shell.execute_reply":"2024-12-27T00:27:58.086315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train= pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ndf_test = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\ndf_train.shape , df_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T00:18:11.006962Z","iopub.execute_input":"2024-12-27T00:18:11.007216Z","iopub.status.idle":"2024-12-27T00:18:16.395463Z","shell.execute_reply.started":"2024-12-27T00:18:11.007194Z","shell.execute_reply":"2024-12-27T00:18:16.394578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#preprocess\ndef preprocess(df_train, df_test):\n    \n    # Separate target and features\n    y = df_train['Premium Amount']\n    X = df_train.drop(['Premium Amount', 'id'], axis=1)\n    y = np.log1p(y)\n    df_test = df_test.drop('id', axis=1)\n    \n    # Process dates\n    for df in [X, df_test]:\n        if 'Policy Start Date' in df.columns:\n            df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n            df['Start Year'] = df['Policy Start Date'].dt.year\n            df['Start Month'] = df['Policy Start Date'].dt.month\n            df['Start Day'] = df['Policy Start Date'].dt.day\n            df['Start Month_sin'] = np.sin(2 * np.pi * df['Start Month']/12)\n            df['Start Month_cos'] = np.cos(2 * np.pi * df['Start Month']/12)\n            df['Start Day_sin'] = np.sin(2 * np.pi * df['Start Day']/31)\n            df['Start Day_cos'] = np.cos(2 * np.pi * df['Start Day']/31)\n            df.drop('Policy Start Date', axis=1, inplace=True)\n    \n    # Define features for imputation\n    categorical_nan = ['Marital Status', 'Occupation', 'Customer Feedback']\n    numerical_nan = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score',\n                    'Previous Claims', 'Credit Score', 'Insurance Duration', 'Vehicle Age']\n    \n    # Handle categorical missing values\n    cat_imputer = SimpleImputer(strategy='constant', fill_value='Unknown')\n    X[categorical_nan] = cat_imputer.fit_transform(X[categorical_nan])\n    df_test[categorical_nan] = cat_imputer.transform(df_test[categorical_nan])\n    \n    # Handle numerical missing values\n    num_imputer = SimpleImputer(strategy='median')\n    X[numerical_nan] = num_imputer.fit_transform(X[numerical_nan])\n    df_test[numerical_nan] = num_imputer.transform(df_test[numerical_nan])\n    \n    # Label encode categorical features\n    categorical = ['Gender', 'Marital Status', 'Education Level', 'Occupation',\n                  'Location', 'Policy Type', 'Customer Feedback', 'Smoking Status',\n                  'Exercise Frequency', 'Property Type']\n    \n    le = LabelEncoder()\n    for col in categorical:\n        # Combine train and test data for fitting\n        combined_values = pd.concat([X[col], df_test[col]])\n        le.fit(combined_values)\n        \n        # Transform separately\n        X[col] = le.transform(X[col])\n        df_test[col] = le.transform(df_test[col])\n    \n    # Scale all features\n    scaler = StandardScaler()\n    \n    # Save column names for later\n    column_names = X.columns\n    \n    # Fit and transform training data\n    X_scaled = scaler.fit_transform(X)\n    X = pd.DataFrame(X_scaled, columns=column_names)\n    \n    # Transform test data\n    test_scaled = scaler.transform(df_test)\n    df_test = pd.DataFrame(test_scaled, columns=column_names)\n    \n    return X, y, df_test\n\ndef preprocess_catboost(df_train, df_test):\n    # Separate target and features\n    y = df_train['Premium Amount']\n    X = df_train.drop(['Premium Amount', 'id'], axis=1)\n    y = np.log1p(y)  # Log transform target\n    df_test = df_test.drop('id', axis=1)\n    \n    # Process dates\n    for df in [X, df_test]:\n        if 'Policy Start Date' in df.columns:\n            df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n            df['Start Year'] = df['Policy Start Date'].dt.year\n            df['Start Month'] = df['Policy Start Date'].dt.month\n            df['Start Day'] = df['Policy Start Date'].dt.day\n            df['Start Month_sin'] = np.sin(2 * np.pi * df['Start Month']/12)\n            df['Start Month_cos'] = np.cos(2 * np.pi * df['Start Month']/12)\n            df['Start Day_sin'] = np.sin(2 * np.pi * df['Start Day']/31)\n            df['Start Day_cos'] = np.cos(2 * np.pi * df['Start Day']/31)\n            df.drop('Policy Start Date', axis=1, inplace=True)\n    \n    # Define feature groups\n    categorical_nan = ['Marital Status', 'Occupation', 'Customer Feedback']\n    numerical_nan = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score',\n                    'Previous Claims', 'Credit Score', 'Insurance Duration', 'Vehicle Age']\n    categorical_features = ['Education Level', 'Customer Feedback', 'Marital Status',\n                          'Policy Type', 'Location', 'Property Type', 'Exercise Frequency']\n    \n    # First handle missing values\n    # For categorical features\n    for col in categorical_nan:\n        if col in X.columns:  # Check if column exists\n            X[col] = X[col].fillna('Missing')\n            df_test[col] = df_test[col].fillna('Missing')\n    \n    # For numerical features\n    numerical_nan = [col for col in numerical_nan if col in X.columns]  # Filter existing columns\n    num_imputer = SimpleImputer(strategy='median')\n    X[numerical_nan] = num_imputer.fit_transform(X[numerical_nan])\n    df_test[numerical_nan] = num_imputer.transform(df_test[numerical_nan])\n    \n    # Convert categorical features to string\n    categorical_features = [col for col in categorical_features if col in X.columns]  # Filter existing columns\n    for col in categorical_features:\n        # First convert to string explicitly\n        X[col] = X[col].astype(str)\n        df_test[col] = df_test[col].astype(str)\n        \n        # Handle any float-like strings by removing decimals\n        X[col] = X[col].apply(lambda x: str(int(float(x))) if x.replace('.','',1).isdigit() else x)\n        df_test[col] = df_test[col].apply(lambda x: str(int(float(x))) if x.replace('.','',1).isdigit() else x)\n    \n    # Important features selection\n    important_features = [\n        'Annual Income', 'Health Score', 'Credit Score', 'Age', 'Vehicle Age', \n        'Insurance Duration', 'Start Day_sin', 'Start Month', 'Start Year',\n        'Number of Dependents', 'Previous Claims', 'Education Level', \n        'Exercise Frequency', 'Customer Feedback', 'Marital Status',\n        'Policy Type', 'Location', 'Property Type'\n    ]\n    \n    # Select important features\n    X = X[important_features]\n    df_test = df_test[important_features]\n    \n    # Get categorical feature indices AFTER feature selection\n    categorical_features = [col for col in categorical_features if col in important_features]\n    cat_features_idx = [important_features.index(col) for col in categorical_features]\n    \n    print(\"\\nFinal column order:\", X.columns.tolist())\n    print(\"Categorical features:\", categorical_features)\n    print(\"Categorical feature indices:\", cat_features_idx)\n    \n    return X, y, df_test, cat_features_idx","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T00:18:16.396813Z","iopub.execute_input":"2024-12-27T00:18:16.397153Z","iopub.status.idle":"2024-12-27T00:18:16.414157Z","shell.execute_reply.started":"2024-12-27T00:18:16.397116Z","shell.execute_reply":"2024-12-27T00:18:16.413303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#apply preprocessing\nX, y, test = preprocess(df_train, df_test)\nX_cat, y, test_cat, cat_features = preprocess_catboost(df_train, df_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T00:18:17.209274Z","iopub.execute_input":"2024-12-27T00:18:17.209592Z","iopub.status.idle":"2024-12-27T00:18:30.668408Z","shell.execute_reply.started":"2024-12-27T00:18:17.209563Z","shell.execute_reply":"2024-12-27T00:18:30.667409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Feature Importance\nimportant_features = [\n    'Annual Income', 'Health Score', 'Credit Score', 'Age', 'Vehicle Age', \n    'Insurance Duration', 'Start Day_sin', 'Start Month', 'Start Year',\n    'Number of Dependents', 'Previous Claims', 'Education Level', \n    'Exercise Frequency', 'Customer Feedback', 'Marital Status',\n    'Policy Type', 'Location', 'Property Type'\n]\n\n# Select important features from the datasets\nX = X[important_features]\ntest = test[important_features]\nX_cat = X_cat[important_features]\ntest_cat = test_cat[important_features]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T00:18:30.669666Z","iopub.execute_input":"2024-12-27T00:18:30.669944Z","iopub.status.idle":"2024-12-27T00:18:30.936594Z","shell.execute_reply.started":"2024-12-27T00:18:30.66992Z","shell.execute_reply":"2024-12-27T00:18:30.935668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ncategorical = ['Gender', 'Marital Status', 'Education Level', 'Location', 'Policy Type', \n               'Customer Feedback', 'Smoking Status', 'Exercise Frequency', \n               'Property Type', 'Occupation']\n\n# Numerical features\nnumerical = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score', \n             'Previous Claims', 'Vehicle Age', 'Credit Score', 'Insurance Duration', \n             'Start Day', 'Start Month', 'Start Year', 'Start Month_sin', \n             'Start Month_cos', 'Start Day_sin', 'Start Day_cos']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T00:18:30.938005Z","iopub.execute_input":"2024-12-27T00:18:30.938298Z","iopub.status.idle":"2024-12-27T00:18:30.942396Z","shell.execute_reply.started":"2024-12-27T00:18:30.938266Z","shell.execute_reply":"2024-12-27T00:18:30.941563Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Stacking Function\n","metadata":{}},{"cell_type":"code","source":"from sklearn.base import clone\n\ndef get_predictions(model_class, model_params, X, y, test, do_oof=True, cat_features=None, n_splits=5, model_name=\"model\"):\n    test_preds = np.zeros(len(test))\n    \n    # First, create base model instance\n    if isinstance(model_class, type):\n        base_model = model_class(**model_params)\n    else:\n        base_model = model_class\n        base_model.set_params(**model_params)\n    \n    if do_oof:\n        # Initialize OOF predictions array\n        oof_preds = np.zeros(len(X))\n        kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n        \n        for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n            print(f\"Training fold {fold + 1}/{n_splits}...\")\n            \n            # Split data for this fold\n            X_train_fold, X_val_fold = X.iloc[train_idx], X.iloc[val_idx]\n            y_train_fold, y_val_fold = y.iloc[train_idx], y.iloc[val_idx]\n            \n            # Create a fresh clone of the base model for this fold\n            model = clone(base_model)\n            \n            # Train the model\n            if isinstance(model, CatBoostRegressor):\n                model.fit(\n                    X_train_fold, \n                    y_train_fold,\n                    cat_features=cat_features,\n                    eval_set=(X_val_fold, y_val_fold),\n                    verbose=False\n                )\n            else:\n                model.fit(X_train_fold, y_train_fold)\n            \n            # Make predictions\n            val_preds = model.predict(X_val_fold)\n            test_preds_fold = model.predict(test)\n            \n            # Store predictions\n            oof_preds[val_idx] = val_preds\n            test_preds += test_preds_fold / n_splits\n            \n        # Transform and save OOF predictions\n        oof_preds_transformed = oof_preds.flatten()\n        oof_df = pd.DataFrame({'Predictions': oof_preds_transformed})\n        oof_df.to_csv(f'oof_preds_{model_name}.csv', index=False)\n        print(f\"OOF predictions saved to oof_preds_{model_name}.csv\")\n        \n    else:\n        # Single model training mode\n        print(\"Training single model...\")\n        if isinstance(base_model, CatBoostRegressor):\n            base_model.fit(\n                X, \n                y,\n                cat_features=cat_features,\n                verbose=False\n            )\n        else:\n            base_model.fit(X, y)\n        \n        test_preds = base_model.predict(test)\n        oof_preds_transformed = None\n\n    # Transform and save test predictions\n    test_preds_transformed = test_preds.flatten()\n    test_df = pd.DataFrame({'Predictions': test_preds_transformed})\n    test_df.to_csv(f'test_preds_{model_name}.csv', index=False)\n    print(f\"Test predictions saved to test_preds_{model_name}.csv\")\n    \n    return oof_preds_transformed, test_preds_transformed","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T20:00:56.268151Z","iopub.execute_input":"2024-12-26T20:00:56.268436Z","iopub.status.idle":"2024-12-26T20:00:56.277187Z","shell.execute_reply.started":"2024-12-26T20:00:56.268412Z","shell.execute_reply":"2024-12-26T20:00:56.276143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_nn_predictions(X, y, test, create_model_fn, model_name=\"ann\", n_splits=5, do_oof=True):\n    test_preds = np.zeros(len(test))\n\n    if do_oof:\n        # OOF prediction mode\n        print(\"Generating OOF predictions...\")\n        oof_preds = np.zeros(len(X))\n        kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n        for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n            print(f\"\\nTraining fold {fold + 1}/{n_splits}\")\n\n            # Split data\n            X_train_fold, X_val_fold = X[train_idx], X[val_idx]\n            y_train_fold, y_val_fold = y[train_idx], y[val_idx]\n\n            # Create and train model\n            model = create_model_fn()\n            callbacks = [\n                tf.keras.callbacks.EarlyStopping(\n                    monitor='val_loss',\n                    patience=10,\n                    restore_best_weights=True\n                ),\n                tf.keras.callbacks.ReduceLROnPlateau(\n                    monitor='val_loss',\n                    factor=0.2,\n                    patience=5,\n                    min_delta=1e-4,\n                    min_lr=1e-7\n                )\n            ]\n\n            history = model.fit(\n                X_train_fold, y_train_fold,\n                validation_data=(X_val_fold, y_val_fold),\n                epochs=200,\n                batch_size=256,\n                callbacks=callbacks,\n                shuffle=True,\n                verbose=1\n            )\n\n            # Generate predictions\n            val_preds = model.predict(X_val_fold, batch_size=256, verbose=0)\n            test_preds_fold = model.predict(test, batch_size=256, verbose=0)\n\n            # Store predictions\n            oof_preds[val_idx] = val_preds.flatten()\n            test_preds += test_preds_fold.flatten() / n_splits\n\n            # Clear session to free memory\n            tf.keras.backend.clear_session()\n\n        # Transform and save OOF predictions\n        oof_preds_transformed = oof_preds.flatten() \n        oof_df = pd.DataFrame({'Predictions': oof_preds_transformed})\n        oof_df.to_csv(f'oof_preds_{model_name}.csv', index=False)\n        print(f\"\\nOOF predictions saved to oof_preds_{model_name}.csv\")\n\n    else:\n        # Single model mode\n        print(\"Training single model...\")\n        model = create_model_fn()\n        callbacks = [\n            tf.keras.callbacks.EarlyStopping(\n                monitor='val_loss',\n                patience=10,\n                restore_best_weights=True\n            ),\n            tf.keras.callbacks.ReduceLROnPlateau(\n                monitor='val_loss',\n                factor=0.2,\n                patience=5,\n                min_delta=1e-4,\n                min_lr=1e-7\n            )\n        ]\n\n        history = model.fit(\n            X, y,\n            validation_split=0.1,\n            epochs=200,\n            batch_size=256,\n            callbacks=callbacks,\n            shuffle=True,\n            verbose=1\n        )\n\n        test_preds = model.predict(test, batch_size=256, verbose=0)\n        oof_preds_transformed = None\n\n    # Transform and save test predictions\n    test_preds_transformed = test_preds.flatten() \n    test_df = pd.DataFrame({'Predictions': test_preds_transformed})\n    test_df.to_csv(f'test_preds_{model_name}.csv', index=False)\n    print(f\"Test predictions saved to test_preds_{model_name}.csv\")\n\n    return oof_preds_transformed, test_preds_transformed","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T21:13:14.056136Z","iopub.execute_input":"2024-12-26T21:13:14.056454Z","iopub.status.idle":"2024-12-26T21:13:14.066231Z","shell.execute_reply.started":"2024-12-26T21:13:14.056424Z","shell.execute_reply":"2024-12-26T21:13:14.065274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\ncat_features = [0, 1, 2]  # indices of categorical features\noof_preds, test_preds = get_oof_and_total_predictions(\n    CatBoostRegressor,\n    model_params,\n    X,\n    y,\n    test,\n    do_oof=True,\n    cat_features=cat_features,\n    model_name=\"catboost\"\n)\n\n# For other models (LightGBM, XGBoost, etc.):\n_, test_preds = get_oof_and_total_predictions(\n    LGBMRegressor,\n    model_params,\n    X,\n    y,\n    do_oof=False,\n    test,\n    model_name=\"lightgbm\"\n)\n'''","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ANN","metadata":{}},{"cell_type":"code","source":"def create_optimized_model():\n    model = Sequential([\n        # Input layer\n        tf.keras.layers.Input(shape=(18,)),\n        Dense(\n            units=34,\n            activation='selu',\n            kernel_regularizer=regularizers.l2(0.00013205)\n        )\n    ])\n    \n    # Layer 1\n    model.add(Dense(\n        units=352,\n        activation='relu',\n        kernel_regularizer=regularizers.l2(0.008567)\n    ))\n    model.add(Dropout(rate=0.18819))  \n    model.add(BatchNormalization())\n    \n    # Layer 2\n    model.add(Dense(\n        units=458,\n        activation='relu',\n        kernel_regularizer=regularizers.l2(0.0021558)\n    ))\n    model.add(BatchNormalization())\n    \n    # Layer 3\n    model.add(Dense(\n        units=264,\n        activation='relu',\n        kernel_regularizer=regularizers.l2(0.0015498)\n    ))\n    model.add(BatchNormalization())\n    \n    # Layer 4\n    model.add(Dense(\n        units=76,\n        activation='elu',\n        kernel_regularizer=regularizers.l2(0.0066342)\n    ))\n    model.add(Dropout(rate=0.39836)) \n    model.add(BatchNormalization())\n    \n    # Layer 5\n    model.add(Dense(\n        units=244,\n        activation='selu',\n        kernel_regularizer=regularizers.l2(0.0057341)\n    ))\n    model.add(Dropout(rate=0.4445))  # dropout_4\n    \n    # Output layer\n    model.add(Dense(1, activation='linear'))\n    \n    # Compile the model\n    model.compile(\n        optimizer='rmsprop',\n        loss=tf.keras.losses.Huber(),\n        metrics=['mean_squared_logarithmic_error']\n    )\n    \n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T21:13:07.105338Z","iopub.execute_input":"2024-12-26T21:13:07.10569Z","iopub.status.idle":"2024-12-26T21:13:07.112574Z","shell.execute_reply.started":"2024-12-26T21:13:07.10566Z","shell.execute_reply":"2024-12-26T21:13:07.111696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#callbacks and fitting\ncallbacks = [\n    tf.keras.callbacks.EarlyStopping(\n        monitor='val_mean_squared_logarithmic_error',  \n        patience=10,                    # Increased patience to give model more time to converge\n        restore_best_weights=True       # This ensures we keep the best performing model\n    ),\n    tf.keras.callbacks.ReduceLROnPlateau(\n        monitor='val_mean_squared_logarithmic_error',\n        factor=0.2,                     # Reduces learning rate by 20% when triggered\n        patience=5,                     # Increased patience for learning rate reduction\n        min_delta=1e-4,                # Minimum change to qualify as an improvement\n        min_lr=1e-7                    # Lower minimum learning rate for fine-tuning\n    )\n]\n\nhistory = ann_model.fit(\n    X, y,                                # Full dataset\n    validation_split=0.1,                # Increased validation split for better evaluation\n    epochs=200,                          # Sufficient epochs for convergence\n    batch_size=256,                      # Balanced batch size for stability\n    callbacks=callbacks,                 \n    shuffle=True,                        # Important to shuffle data before splitting\n    verbose=1                           \n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:12:57.692917Z","iopub.execute_input":"2024-12-26T15:12:57.693236Z","iopub.status.idle":"2024-12-26T15:12:57.697624Z","shell.execute_reply.started":"2024-12-26T15:12:57.693207Z","shell.execute_reply":"2024-12-26T15:12:57.696646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate predictions\npredictions = np.expm1(ann_model.predict(test, batch_size=256, verbose=1))\n\npredictions_df = pd.DataFrame({\n    'Predictions': predictions_transformed.flatten()  # flatten() converts from 2D to 1D array\n})\n\n# Export to CSV\npredictions_df.to_csv('ann_pred.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## XGB","metadata":{}},{"cell_type":"code","source":"#optuna\nX_train, X_val, y_train, y_val = train_test_split(X_new, y, test_size=0.15, random_state=42)\n\n# Objective function to optimize XGBoost with RMSLE\ndef objective(trial):\n    param = {\n        'max_depth': trial.suggest_int('max_depth', 3,7),\n        'n_estimators': trial.suggest_int('n_estimators', 80, 120),\n        'reg_lambda': trial.suggest_float('reg_lambda', 0, 3),\n        'reg_alpha': trial.suggest_float('reg_alpha', 0, 1.0),\n        'learning_rate': trial.suggest_float('learning_rate', 0, 0.2),\n        # 'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n        # 'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),\n        # 'objective': 'reg:squarederror',\n        # 'eval_metric': 'rmse',\n        'tree_method': 'hist',\n        'device': 'cuda',\n        'predictor': 'gpu_predictor',\n        'n_jobs': -1,\n        'verbose': 1\n    }\n    model = XGBRegressor(use_label_encoder=False, **param)\n    model.fit(X_train, y_train)\n    preds = model.predict(X_val)\n    rmsle = np.sqrt(mean_squared_log_error(np.expm1(y_val), np.expm1(np.maximum(preds,0))))\n    return rmsle\n\n# Create study\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=100)\n\n# Best hyperparameters\nbest_params = study.best_params\nprint(f'Best Hyperparameters: {best_params}')\n\njoblib.dump(best_params, '/kaggle/working/best_params_xgb1.pkl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T23:20:05.934284Z","iopub.execute_input":"2024-12-26T23:20:05.934628Z","iopub.status.idle":"2024-12-26T23:23:30.401614Z","shell.execute_reply.started":"2024-12-26T23:20:05.934597Z","shell.execute_reply":"2024-12-26T23:23:30.400731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#model fitting\nxgb_model = XGBRegressor(**best_params)\nxgb_model.fit(X_new, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T19:08:00.036398Z","iopub.execute_input":"2024-12-26T19:08:00.036736Z","iopub.status.idle":"2024-12-26T19:08:06.215919Z","shell.execute_reply.started":"2024-12-26T19:08:00.036709Z","shell.execute_reply":"2024-12-26T19:08:06.214458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate predictions\npredictions = np.expm1(xgb_model.predict(test_new))\n\npredictions_df = pd.DataFrame({\n    'Predictions': predictions.flatten()  # flatten() converts from 2D to 1D array\n})\n\n# Export to CSV\njoblib.dump(predictions_df, '/kaggle/working/preds.pkl')\npredictions_df.to_csv('pred.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T19:13:42.250749Z","iopub.execute_input":"2024-12-26T19:13:42.251187Z","iopub.status.idle":"2024-12-26T19:13:43.883667Z","shell.execute_reply.started":"2024-12-26T19:13:42.251148Z","shell.execute_reply":"2024-12-26T19:13:43.882921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Stack\noof_preds, test_preds = get_oof_and_total_predictions(\n    XGBRegressor,\n    best_params,\n    X,\n    y,\n    test,\n    do_oof=True,\n    model_name=\"xgboost\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SGD","metadata":{}},{"cell_type":"code","source":"\nX_train, X_val, y_train, y_val = train_test_split(X_new, y, test_size=0.15, random_state=42)\n\n# Define the objective function for Optuna\ndef objective(trial):\n    # Define hyperparameters to tune\n    params = {\n        'alpha': trial.suggest_float('alpha', 1e-5, 1e-1),         # Regularization term\n        'learning_rate': trial.suggest_categorical('learning_rate', ['constant', 'optimal', 'invscaling', 'adaptive']),\n        'eta0': trial.suggest_float('eta0', 1e-4, 1e-1),          # Initial learning rate\n        'penalty': trial.suggest_categorical('penalty', ['l2', 'l1', 'elasticnet']),\n        'max_iter': trial.suggest_int('max_iter', 500, 5000),      # Maximum number of iterations\n        'random_state': 42\n    }\n\n    # Train the SGD Regressor\n    sgd = SGDRegressor(**params)\n    sgd.fit(X_train, y_train)\n\n    # Predict and calculate RMSLE\n    preds = sgd.predict(X_test)\n    preds = np.maximum(preds, 0)  # Ensure non-negative predictions\n    rmsle = np.sqrt(mean_squared_log_error(np.expm1(y_test), np.expm1(preds)))\n\n    return rmsle\n\n# Optimize hyperparameters using Optuna\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=100)\n\nbest_params = study.best_params\n\n# Output the best parameters and RMSLE\nprint(\"Best parameters:\", study.best_params)\nprint(\"Best RMSLE:\", study.best_value)\n\n# Save the best parameters for reuse\njoblib.dump(study.best_params, '/kaggle/working/best_params_sgd1.pkl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T23:25:44.08587Z","iopub.execute_input":"2024-12-26T23:25:44.086182Z","iopub.status.idle":"2024-12-26T23:40:30.111742Z","shell.execute_reply.started":"2024-12-26T23:25:44.086157Z","shell.execute_reply":"2024-12-26T23:40:30.110178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#model fitting\nsgd_model = SGDRegressor(**best_params)\nsgd_model.fit(X_train, y_train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate predictions\npredictions = np.expm1(sgd_model.predict(X_test))\n\npredictions_df = pd.DataFrame({\n    'Predictions': predictions_transformed.flatten()  # flatten() converts from 2D to 1D array\n})\n\n# Export to CSV\npredictions_df.to_csv('sgd_pred.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Stack\noof_preds, test_preds = get_oof_and_total_predictions(\n    SGDRegressor,\n    best_params,\n    X,\n    y,\n    test,\n    do_oof=True,\n    model_name=\"sgd\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LGBM","metadata":{}},{"cell_type":"code","source":"# Split data into training and testing sets\nX_train, X_val, y_train, y_val = train_test_split(X_new, y, test_size=0.15, random_state=42)\n\n# Define the objective function for Optuna\ndef objective(trial):\n    # Define hyperparameters to tune\n    params = {\n        'objective': 'regression',                      # Regression task\n        'metric': 'rmse',                               # Evaluation metric\n        'boosting_type': 'gbdt',                        # Gradient Boosted Decision Trees\n        'max_depth': trial.suggest_int('max_depth', 3, 10),              # Max depth of trees\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3), # Learning rate\n        'n_estimators': trial.suggest_int('n_estimators', 100, 1000),     # Number of estimators\n        'subsample': trial.suggest_float('subsample', 0.5, 1.0),          # Fraction of data used per iteration\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0), # Fraction of features used per iteration\n        'lambda_l1': trial.suggest_float('lambda_l1', 1e-8, 10.0),        # L1 regularization term\n        'lambda_l2': trial.suggest_float('lambda_l2', 1e-8, 10.0),        # L2 regularization term\n        'device': 'gpu',                                # Use GPU for training\n        'verbose': -1                                   # Suppress training output\n    }\n\n    # Train the LightGBM Regressor\n    model = LGBMRegressor(**params)\n    model.fit(X_train, y_train)\n\n    # Predict and calculate RMSLE\n    preds = model.predict(X_val)\n    preds = np.maximum(preds, 0)  # Ensure non-negative predictions\n    rmsle = np.sqrt(mean_squared_log_error(np.expm1(y_val), np.expm1(preds)))\n\n    return rmsle\n\n# Optimize hyperparameters using Optuna\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)\n\nbest_params = study.best_params\n\n# Output the best parameters and RMSLE\nprint(\"Best parameters:\", study.best_params)\nprint(\"Best RMSLE:\", study.best_value)\n\n# Save the best parameters for reuse\njoblib.dump(study.best_params, '/kaggle/working/best_params_lgb1.pkl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T23:41:07.921277Z","iopub.execute_input":"2024-12-26T23:41:07.921657Z","iopub.status.idle":"2024-12-26T23:51:57.239508Z","shell.execute_reply.started":"2024-12-26T23:41:07.921624Z","shell.execute_reply":"2024-12-26T23:51:57.238664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#model fitting\nlgbm_model = LGBMRegressor(**best_params)\nlgbm_model.fit(X_train, y_train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate predictions\npredictions = np.expm1(lgbm_model.predict(X_test))\n\npredictions_df = pd.DataFrame({\n    'Predictions': predictions_transformed.flatten()  # flatten() converts from 2D to 1D array\n})\n\n# Export to CSV\npredictions_df.to_csv('lgbm_pred.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Stack\noof_preds, test_preds = get_oof_and_total_predictions(\n    LGBMRegressor,\n    best_params,\n    X,\n    y,\n    test,\n    do_oof=True,\n    model_name=\"lgbm\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CatBoost","metadata":{}},{"cell_type":"code","source":"X_cat.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T14:59:16.384174Z","iopub.execute_input":"2024-12-26T14:59:16.384526Z","iopub.status.idle":"2024-12-26T14:59:16.731422Z","shell.execute_reply.started":"2024-12-26T14:59:16.384486Z","shell.execute_reply":"2024-12-26T14:59:16.730551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_columns = X_cat.columns[cat_features].tolist()\n\n# Convert categorical features to string\nfor col in categorical_columns:\n    X_cat[col] = X_cat[col].astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T14:59:10.293711Z","iopub.execute_input":"2024-12-26T14:59:10.294036Z","iopub.status.idle":"2024-12-26T14:59:10.478345Z","shell.execute_reply.started":"2024-12-26T14:59:10.294006Z","shell.execute_reply":"2024-12-26T14:59:10.477398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Split the data\nX_train, X_val, y_train, y_val = train_test_split(X_cat, y, test_size=0.15, random_state=42)\ndef objective(trial):\n    params = {\n        'task_type': 'GPU',\n        'devices': '0:1',  # Use both GPUs\n        'gpu_ram_part': 0.95,  # Use 95% of GPU memory\n        \n        # Learning parameters\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1),\n        'iterations': trial.suggest_int('iterations', 500, 3000),\n        \n        # Tree parameters\n        'depth': trial.suggest_int('depth', 5, 8),\n        'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1e-3, 10.0),\n        'random_strength': trial.suggest_float('random_strength', 1e-3, 10.0),\n        \n        # Loss function\n        'loss_function': 'RMSE',\n        \n        # Categorical features\n        'one_hot_max_size': trial.suggest_int('one_hot_max_size', 10, 100),\n        \n        # Other parameters\n        'leaf_estimation_method': 'Newton',\n        'bootstrap_type': trial.suggest_categorical('bootstrap_type', ['Bernoulli']),\n        \n        # Performance parameters\n        'thread_count': -1,\n        'verbose': False,  # Set to False to avoid CatBoost logs\n        'random_seed': 42\n    }\n    \n    if params['bootstrap_type'] == 'Bernoulli':\n        params['subsample'] = trial.suggest_float('subsample', 0.5, 0.8)\n    \n    early_stopping_rounds = trial.suggest_int('early_stopping_rounds', 30, 100)\n    \n    try:\n        # Initialize and train model\n        model = CatBoostRegressor(**params)\n        \n        # Fit with early stopping\n        model.fit(\n            X_train,\n            y_train,\n            cat_features=cat_features,\n            eval_set=[(X_val, y_val)],\n            early_stopping_rounds=early_stopping_rounds,\n            verbose=False,  # Set to False to avoid CatBoost logs\n            use_best_model=True\n        )\n        \n        # Make predictions\n        preds = model.predict(X_val)\n        \n        # Calculate RMSLE\n        rmsle = np.sqrt(mean_squared_log_error(np.expm1(y_val), np.maximum(0, np.expm1(preds))))\n        \n        return rmsle\n    \n    except Exception as e:\n        print(f\"Trial failed with error: {str(e)}\")\n        return float('inf')\n\n# Callback to print trial results\ndef print_callback(study, trial):\n    if trial.number > 0:\n        print(f'\\nTrial {trial.number}:')\n        print(f'  Value (RMSLE): {trial.value:.4f}')\n        print(f'  Best value so far: {study.best_value:.4f}')\n        print(f'  Parameters: {trial.params}')\n\n# Create study\nstudy = optuna.create_study(\n    direction='minimize',\n    pruner=optuna.pruners.MedianPruner(\n        n_startup_trials=5,\n        n_warmup_steps=20,\n        interval_steps=10\n    )\n)\n\n# Optimize with callback\nstudy.optimize(\n    objective, \n    n_trials=100,\n    callbacks=[print_callback],\n    show_progress_bar=True\n)\n\n# Print final results\nprint(\"\\nStudy completed!\")\nprint(f\"\\nBest trial:\")\ntrial = study.best_trial\nprint(f\"  Value (RMSLE): {trial.value:.4f}\")\nprint(\"\\nBest parameters:\")\nfor key, value in trial.params.items():\n    print(f\"  {key}: {value}\")\n\n# Save best parameters\njoblib.dump(study.best_params, 'best_params_catboost1.pkl')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Level 2\n","metadata":{}},{"cell_type":"code","source":"#Checking best results\n\nxgb_params = joblib.load('/kaggle/input/best-params/best_params_xgb1.pkl')\nlgb_params = joblib.load('/kaggle/input/best-params/best_params_lgb1.pkl')\nsgd_params = joblib.load('/kaggle/input/best-params/best_params_sgd1 (2).pkl')\ncatboost_params = joblib.load('/kaggle/input/best-params/best_params_catboost1.pkl')\n\n\nmodels = {}\n\nmodels = {\n    'XGB': (XGBRegressor, xgb_params),\n    'LGBM': (LGBMRegressor, lgb_params),\n    'SGD': (SGDRegressor, sgd_params),\n    'CAT': (CatBoostRegressor, catboost_params),\n    # 'ann' : (ann_model , xgb_params)\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T00:53:33.134582Z","iopub.execute_input":"2024-12-27T00:53:33.134913Z","iopub.status.idle":"2024-12-27T00:53:33.151639Z","shell.execute_reply.started":"2024-12-27T00:53:33.134883Z","shell.execute_reply":"2024-12-27T00:53:33.150958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split the data indices\nX_train, X_test, y_train, y_test, train_indices, test_indices = train_test_split(\n    X_new, y, range(len(X_new)), test_size=0.2, random_state=42)\n\n# Use the indices to split X_cat\nX_train_cat = X_cat.iloc[train_indices]\nX_test_cat = X_cat.iloc[test_indices]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T00:54:10.458986Z","iopub.execute_input":"2024-12-27T00:54:10.459285Z","iopub.status.idle":"2024-12-27T00:54:11.335894Z","shell.execute_reply.started":"2024-12-27T00:54:10.459259Z","shell.execute_reply":"2024-12-27T00:54:11.335134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#checking\nrmsle_scores = []\n\nfor model_name, (model_class, params) in models.items():\n    if model_name == 'ann':\n        oof_preds, test_predictions = get_nn_predictions(\n            X=X.values,\n            y=y.values,\n            test=test.values,\n            create_model_fn=create_optimized_model,\n            model_name=\"ann\",\n            n_splits=5,\n            do_oof=True\n        )\n        \n    elif model_name == 'CAT':\n        oof_preds, test_preds = get_predictions(\n            model_class,\n            params,\n            X=X_cat,\n            y=y,\n            test=test_cat,\n            do_oof = True,\n            cat_features=cat_features,\n            model_name=model_name\n        )\n    else:\n        oof_preds, test_preds = get_predictions(\n            model_class,\n            params,\n            X=X_new,\n            y=y,\n            test=test_new,\n            do_oof=True,\n            model_name=model_name\n        )\n\n    \n#     rmsle = np.sqrt(mean_squared_log_error(np.expm1(y_test), np.expm1(test_preds)))\n#     rmsle_scores.append(rmsle)\n#     print(f\"{model_name} RMSLE: {rmsle}\")\n\n\n# print(\"\\nAll RMSLE scores:\", rmsle_scores)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T00:59:16.849286Z","iopub.execute_input":"2024-12-27T00:59:16.849623Z","iopub.status.idle":"2024-12-27T01:05:44.750488Z","shell.execute_reply.started":"2024-12-27T00:59:16.849595Z","shell.execute_reply":"2024-12-27T01:05:44.749561Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#sofd\ndf = pd.read_csv('/kaggle/working/test_preds_ann.csv')\nrmsle = np.sqrt(mean_squared_log_error(np.expm1(y_test), np.maximum(0,df)))\nrmsle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T17:55:38.757826Z","iopub.execute_input":"2024-12-26T17:55:38.758113Z","iopub.status.idle":"2024-12-26T17:55:38.798065Z","shell.execute_reply.started":"2024-12-26T17:55:38.758089Z","shell.execute_reply":"2024-12-26T17:55:38.797355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_predictions, test_preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T01:39:31.834884Z","iopub.execute_input":"2024-12-27T01:39:31.835211Z","iopub.status.idle":"2024-12-27T01:39:31.840816Z","shell.execute_reply.started":"2024-12-27T01:39:31.835184Z","shell.execute_reply":"2024-12-27T01:39:31.839793Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Stacking","metadata":{}},{"cell_type":"code","source":"def create_meta_features(input_path='/kaggle/working/'):\n    # List of models we have predictions for\n    models = ['XGB', 'LGBM', 'SGD', 'CAT' , 'ann']\n    \n    # Initialize DataFrames for meta features\n    meta_train = pd.DataFrame()\n    meta_test = pd.DataFrame()\n    \n    # Read each model's predictions and add them to meta features\n    for model in models:\n        # Read OOF predictions (training meta features)\n        oof_preds = pd.read_csv(f'{input_path}oof_preds_{model}.csv')\n        meta_train[f'{model}_pred'] = oof_preds\n        \n        # Read test predictions\n        test_preds = pd.read_csv(f'{input_path}test_preds_{model}.csv')\n        meta_test[f'{model}_pred'] = test_preds\n    \n    print(\"Meta features shape:\", meta_train.shape)\n    print(\"Meta test features shape:\", meta_test.shape)\n    # print(\"\\nMeta features preview:\")\n    # print(meta_train.head())\n    \n    return meta_train, meta_test\n\n# Create the meta-features\nmeta_features, meta_features_test = create_meta_features()\n\n# If you want to save these organized meta-features\n# meta_features.to_csv('meta_train_features.csv', index=False)\n# meta_features_test.to_csv('meta_test_features.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T01:20:53.92432Z","iopub.execute_input":"2024-12-27T01:20:53.924671Z","iopub.status.idle":"2024-12-27T01:20:55.534119Z","shell.execute_reply.started":"2024-12-27T01:20:53.924641Z","shell.execute_reply":"2024-12-27T01:20:55.533174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split data into training and testing sets\nX_train, X_val, y_train, y_val = train_test_split(meta_features, y, test_size=0.15, random_state=42)\n\n# Define the objective function for Optuna\ndef objective(trial):\n    # Define hyperparameters to tune\n    params = {\n        'objective': 'regression',                      # Regression task\n        'metric': 'rmse',                               # Evaluation metric\n        'boosting_type': 'gbdt',                        # Gradient Boosted Decision Trees\n        'max_depth': trial.suggest_int('max_depth', 3, 10),              # Max depth of trees\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3), # Learning rate\n        'n_estimators': trial.suggest_int('n_estimators', 100, 1000),     # Number of estimators\n        'subsample': trial.suggest_float('subsample', 0.5, 1.0),          # Fraction of data used per iteration\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0), # Fraction of features used per iteration\n        'lambda_l1': trial.suggest_float('lambda_l1', 1e-8, 10.0),        # L1 regularization term\n        'lambda_l2': trial.suggest_float('lambda_l2', 1e-8, 10.0),        # L2 regularization term\n        'device': 'gpu',                                # Use GPU for training\n        'verbose': -1                                   # Suppress training output\n    }\n\n    # Train the LightGBM Regressor\n    model = LGBMRegressor(**params)\n    model.fit(X_train, y_train)\n\n    # Predict and calculate RMSLE\n    preds = model.predict(X_val)\n    preds = np.maximum(preds, 0)  # Ensure non-negative predictions\n    rmsle = np.sqrt(mean_squared_log_error(np.expm1(y_val), np.expm1(preds)))\n\n    return rmsle\n\n# Optimize hyperparameters using Optuna\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)\n\nbest_params = study.best_params\n\n# Output the best parameters and RMSLE\nprint(\"Best parameters:\", study.best_params)\nprint(\"Best RMSLE:\", study.best_value)\n\n# Save the best parameters for reuse\njoblib.dump(study.best_params, '/kaggle/working/best_params_metalgb1.pkl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T01:22:49.35916Z","iopub.execute_input":"2024-12-27T01:22:49.359619Z","iopub.status.idle":"2024-12-27T01:30:02.647872Z","shell.execute_reply.started":"2024-12-27T01:22:49.359575Z","shell.execute_reply":"2024-12-27T01:30:02.647192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#get preds\n# best_params =  joblib.load('/kaggle/input/best-params/best_params_lgbm.pkl')\noof_preds, test_preds = get_predictions(\n            LGBMRegressor,\n            best_params,\n            X=meta_features,\n            y=y,\n            test=meta_features_test,\n            do_oof=True,\n            model_name='meta_lgbm'\n        )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T02:03:54.53405Z","iopub.execute_input":"2024-12-27T02:03:54.534369Z","iopub.status.idle":"2024-12-27T02:05:04.593127Z","shell.execute_reply.started":"2024-12-27T02:03:54.534338Z","shell.execute_reply":"2024-12-27T02:05:04.592073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.DataFrame(np.expm1(test_preds)).to_csv('final.csv' , index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T01:32:26.581853Z","iopub.execute_input":"2024-12-27T01:32:26.582159Z","iopub.status.idle":"2024-12-27T01:32:27.716406Z","shell.execute_reply.started":"2024-12-27T01:32:26.582131Z","shell.execute_reply":"2024-12-27T01:32:27.715296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nX_new = X.copy()\ntest_new = test.copy()\na = pd.read_csv('/kaggle/input/for-stacking/oof_preds_meta_lgbm.csv')\nb = pd.read_csv('/kaggle/input/for-stacking/test_preds_meta_lgbm.csv')\n\nX_cat['best'] = a\ntest_cat['best'] = np.log1p(b)\n\nscaler = StandardScaler()\nX_new['best'] = scaler.fit_transform(a.values)\ntest_new['best'] = scaler.transform(np.log1p(b))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T00:24:37.049685Z","iopub.execute_input":"2024-12-27T00:24:37.049993Z","iopub.status.idle":"2024-12-27T00:24:37.47172Z","shell.execute_reply.started":"2024-12-27T00:24:37.049966Z","shell.execute_reply":"2024-12-27T00:24:37.47077Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Blending","metadata":{}},{"cell_type":"code","source":"test_preds test_predictions","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom scipy.optimize import differential_evolution\nfrom sklearn.metrics import mean_squared_log_error\n\ndef load_model_predictions(model_name):\n   \n    oof_preds = pd.read_csv(f'/kaggle/working/oof_preds_{model_name}.csv').values\n    test_preds = pd.read_csv(f'/kaggle/working/test_preds_{model_name}.csv').values\n    return oof_preds, test_preds\n\ndef weighted_average(weights, predictions):\n   \n    sum_weighted_preds = predictions[0] * weights[0]\n    sum_weights = weights[0]\n    \n    for i in range(1, len(predictions)):\n        sum_weighted_preds += predictions[i] * weights[i]\n        sum_weights += weights[i]\n    \n    return sum_weighted_preds / sum_weights\n\ndef blend_predictions(y_true, models=['LGBM', 'XGB', 'SGD', 'CAT', 'ann']):\n    \n    # Load predictions (all in log space)\n    oof_predictions = []\n    test_predictions = []\n    for model in models:\n        oof_preds, test_preds = load_model_predictions(model)\n        oof_predictions.append(oof_preds)\n        test_predictions.append(test_preds)\n    \n    def optimization_objective(weights):\n        \n        # Get weighted average in log space\n        blended_preds = weighted_average(weights, oof_predictions)\n        \n        # Transform back to original scale for RMSLE calculation\n        original_scale_preds = np.expm1(blended_preds)\n        original_scale_true = np.expm1(y_true)\n        \n        # Clip predictions in original scale\n        clipped_preds = np.clip(original_scale_preds, 20, 4999)\n        \n        return np.sqrt(mean_squared_log_error(original_scale_true, clipped_preds))\n    \n    # Run differential evolution optimization\n    bounds = [(0.0, 10.0) for _ in range(len(models))]\n    result = differential_evolution(\n        optimization_objective,\n        bounds=bounds,\n        maxiter=500,\n        popsize=15,\n        mutation=(0.5, 1),\n        recombination=0.7,\n        seed=42\n    )\n    \n    # Get final predictions using optimized weights\n    weights = result.x\n    train_blend = weighted_average(weights, oof_predictions)  # Still in log space\n    test_blend = weighted_average(weights, test_predictions)  # Still in log space\n    \n    # Calculate final score (transforming back to original scale)\n    final_score = optimization_objective(weights)\n    \n    print(\"\\nOptimized model weights:\")\n    for model, weight in zip(models, weights):\n        print(f\"{model}: {weight:.4f}\")\n    print(f\"\\nFinal RMSLE: {final_score:.6f}\")\n    \n    return weights, train_blend, test_blend\n\n# Run the blending process\nweights, train_predictions, test_predictions = blend_predictions(y)\n\n# Create submission file (transform back to original scale and clip)\n# submission = pd.DataFrame({\n#     'id': range(1200000, 2000000),\n#     'Premium Amount': np.clip(np.expm1(test_predictions), 20, 4999)\n# })\n# submission.to_csv('blended_submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T01:51:04.735596Z","iopub.execute_input":"2024-12-27T01:51:04.735895Z","iopub.status.idle":"2024-12-27T01:51:11.637945Z","shell.execute_reply.started":"2024-12-27T01:51:04.735869Z","shell.execute_reply":"2024-12-27T01:51:11.637224Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#get pred\npd.DataFrame(np.expm1(test_predictions)).to_csv('final.csv' , index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T01:37:24.91463Z","iopub.execute_input":"2024-12-27T01:37:24.914931Z","iopub.status.idle":"2024-12-27T01:37:26.055196Z","shell.execute_reply.started":"2024-12-27T01:37:24.914907Z","shell.execute_reply":"2024-12-27T01:37:26.054394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Modify your data preparation\nmembers = ['LGBM', 'XGB', 'SGD', 'CAT', 'ann']\ntrain_preds = {}\ntest_preds = {}\n\nfor model in members:\n    train_preds[model] = np.log1p(pd.read_csv(f'/kaggle/input/for-stacking/oof_preds_{model}.csv').values)\n    test_preds[model] = np.log1p(pd.read_csv(f'/kaggle/input/for-stacking/test_preds_{model}.csv').values)\n\n# Create prediction matrices in log space\ntrain_df = pd.DataFrame(train_preds)\ntest_df = pd.DataFrame(test_preds)\n\n# Apply their blending\nblended_preds, score = blend_diff(train_df, test_df, members)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T02:01:30.778898Z","iopub.execute_input":"2024-12-27T02:01:30.779186Z","iopub.status.idle":"2024-12-27T02:01:30.78524Z","shell.execute_reply.started":"2024-12-27T02:01:30.779161Z","shell.execute_reply":"2024-12-27T02:01:30.784335Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\n\n# Create dictionary of weights with the provided normalized weights\nweights = {\n    'LGBM': 0.2933,\n    'XGB': 0.1442,\n    'SGD': 0.3123,\n    'CAT': 0.2427,\n    'ANN': 0.0075\n}\n\n# Initialize arrays for predictions\ntest_predictions = np.zeros(len(meta_features_test))\noof_predictions = np.zeros(len(meta_features))\n\n# Create a mapping of column names to match your DataFrame structure\ncolumn_mapping = {\n    'LGBM': 'LGBM_pred',\n    'XGB': 'XGB_pred',\n    'SGD': 'SGD_pred',\n    'CAT': 'CAT_pred',\n    'ANN': 'ann_pred'\n}\n\nprint(\"Starting to combine predictions with the following weights:\")\nfor model, weight in weights.items():\n    print(f\"{model} ({column_mapping[model]}): {weight:.4f}\")\n\n# Calculate weighted predictions using the correct column names\nfor model_name, weight in weights.items():\n    column_name = column_mapping[model_name]\n    # Add weighted predictions for each model\n    oof_predictions += meta_features[column_name].values * weight\n    test_predictions += meta_features_test[column_name].values * weight\n\n# Save predictions to CSV\noof_df = pd.DataFrame({'Predictions': oof_predictions})\noof_df.to_csv('oof_preds_ensemble.csv', index=False)\nprint(\"\\nOOF predictions saved to oof_preds_ensemble.csv\")\n\ntest_df = pd.DataFrame({'Predictions': test_predictions})\ntest_df.to_csv('test_preds_ensemble.csv', index=False)\nprint(\"Test predictions saved to test_preds_ensemble.csv\")\n\nprint(\"\\nShape of final predictions:\")\nprint(f\"OOF predictions shape: {oof_predictions.shape}\")\nprint(f\"Test predictions shape: {test_predictions.shape}\")\n\n# Display some statistics to verify the predictions look reasonable\nprint(\"\\nPrediction Statistics:\")\nprint(\"Test Predictions Summary:\")\nprint(f\"Mean: {test_predictions.mean():.6f}\")\nprint(f\"Std: {test_predictions.std():.6f}\")\nprint(f\"Min: {test_predictions.min():.6f}\")\nprint(f\"Max: {test_predictions.max():.6f}\")\n\n# Let's also show a sample of the weighted predictions to verify they make sense\nprint(\"\\nSample of first few predictions:\")\nsample_idx = 0\nprint(\"\\nDetailed calculation for first row:\")\nfor model_name, weight in weights.items():\n    column_name = column_mapping[model_name]\n    contribution = meta_features_test[column_name].iloc[0] * weight\n    print(f\"{model_name}: {meta_features_test[column_name].iloc[0]:.6f} * {weight:.4f} = {contribution:.6f}\")\nprint(f\"Final weighted prediction: {test_predictions[0]:.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T02:02:25.179359Z","iopub.execute_input":"2024-12-27T02:02:25.179692Z","iopub.status.idle":"2024-12-27T02:02:27.968161Z","shell.execute_reply.started":"2024-12-27T02:02:25.179664Z","shell.execute_reply":"2024-12-27T02:02:27.967246Z"}},"outputs":[],"execution_count":null}]}