{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm\nimport seaborn as sns\nimport math\nfrom sklearn.preprocessing import LabelEncoder, OrdinalEncoder, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\n\nfrom scipy.signal import find_peaks\nfrom scipy.stats import skew \n\nimport optuna\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.preprocessing import StandardScaler\nimport lightgbm as lgb\nfrom lightgbm import early_stopping, log_evaluation\n\n# Ignore general warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Suppress LightGBM logs\nimport logging\nlogging.getLogger(\"lightgbm\").setLevel(logging.ERROR)\n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:56:30.045579Z","iopub.execute_input":"2024-12-30T17:56:30.045825Z","iopub.status.idle":"2024-12-30T17:56:34.09205Z","shell.execute_reply.started":"2024-12-30T17:56:30.045802Z","shell.execute_reply":"2024-12-30T17:56:34.091178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"RETUNE_CATBOOST = False\nRETUNE_LGBM = False\nRETUNE_LASSO = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:56:34.093319Z","iopub.execute_input":"2024-12-30T17:56:34.09394Z","iopub.status.idle":"2024-12-30T17:56:34.097119Z","shell.execute_reply.started":"2024-12-30T17:56:34.093916Z","shell.execute_reply":"2024-12-30T17:56:34.09631Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"background-color: PINK; text-align: center; font-size: 24px; font-weight: bold; color: GREEN\">\nIMPORTING DATA\n</div>","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv').drop(columns='id')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv').drop(columns='id')\n\n# Verify shapes\nprint(\"Train Data Shape:\", train.shape)\nprint(\"Test Data Shape:\", test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:56:34.098978Z","iopub.execute_input":"2024-12-30T17:56:34.099282Z","iopub.status.idle":"2024-12-30T17:56:43.405324Z","shell.execute_reply.started":"2024-12-30T17:56:34.099249Z","shell.execute_reply":"2024-12-30T17:56:43.404583Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"background-color: PINK; text-align: center; font-size: 24px; font-weight: bold; color: GREEN\">\nDATA SUMMARIES\n</div>","metadata":{}},{"cell_type":"code","source":"# Display Sample Data\nprint(\"Training Dataset: \\n\")\ndisplay(train.head())\nprint('\\n')\nprint(\"Test Dataset: \\n\")\ndisplay(test.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:56:43.406405Z","iopub.execute_input":"2024-12-30T17:56:43.406711Z","iopub.status.idle":"2024-12-30T17:56:43.531087Z","shell.execute_reply.started":"2024-12-30T17:56:43.406666Z","shell.execute_reply":"2024-12-30T17:56:43.530266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"TRAIN INFO \\n\\n\")\ndisplay(train.info())\nprint(\"\\n\\n\")\nprint(\"TEST INFO \\n\\n\")\ndisplay(test.info())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:56:43.53187Z","iopub.execute_input":"2024-12-30T17:56:43.53209Z","iopub.status.idle":"2024-12-30T17:56:44.4263Z","shell.execute_reply.started":"2024-12-30T17:56:43.53207Z","shell.execute_reply":"2024-12-30T17:56:44.42545Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Descriptive statistics\ndisplay(train.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:56:44.427179Z","iopub.execute_input":"2024-12-30T17:56:44.427465Z","iopub.status.idle":"2024-12-30T17:56:44.980795Z","shell.execute_reply.started":"2024-12-30T17:56:44.427431Z","shell.execute_reply":"2024-12-30T17:56:44.979941Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"background-color: PINK; text-align: center; font-size: 24px; font-weight: bold; color: BLACK\">\nHANDLING MISSING VALUES\n</div>","metadata":{}},{"cell_type":"code","source":"missing_vals_train = train.isnull()\nmissing_vals_test = test.isnull()\n\n# Create a single figure with subplots for both datasets\nfig, axes = plt.subplots(2, 1, figsize=(18, 12))\n\nsns.heatmap(missing_vals_train, cmap='viridis', cbar=True, yticklabels=False, ax=axes[0])\naxes[0].set_title('Missing Values Heatmap For Training Dataset', fontsize=14)\naxes[0].set_xlabel('Features', fontsize=12)\naxes[0].set_ylabel('Entries', fontsize=12)\n\nsns.heatmap(missing_vals_test, cmap='viridis', cbar=True, yticklabels=False, ax=axes[1])\naxes[1].set_title('Missing Values Heatmap For Test Dataset', fontsize=14)\naxes[1].set_xlabel('Features', fontsize=12)\naxes[1].set_ylabel('Entries', fontsize=12)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:56:44.981643Z","iopub.execute_input":"2024-12-30T17:56:44.982036Z","iopub.status.idle":"2024-12-30T17:57:24.467648Z","shell.execute_reply.started":"2024-12-30T17:56:44.982Z","shell.execute_reply":"2024-12-30T17:57:24.466764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to calculate missing values, percentages, and data types\ndef missing_vals_table(dataframe):\n    missing_count = dataframe.isnull().sum()\n    missing_percentage = missing_count / len(dataframe) * 100\n\n    return pd.DataFrame({\n        'Missing Values': missing_count,\n        'Percentage (%)': missing_percentage,\n        'Datatype': dataframe.dtypes\n    })\n\n# Creating table for train and test values\ntrain_missing_table = missing_vals_table(train)\ntest_missing_table = missing_vals_table(test)\n\n# Display The Tables\nprint('\\n')\nprint(\"Missing vals table - Training data\\n\")\ndisplay(train_missing_table[train_missing_table['Missing Values'] > 0])\nprint('\\n')\nprint(\"Missing vals table - Test data\\n\")\ndisplay(test_missing_table[test_missing_table['Missing Values'] > 0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:57:24.46982Z","iopub.execute_input":"2024-12-30T17:57:24.470072Z","iopub.status.idle":"2024-12-30T17:57:25.371814Z","shell.execute_reply.started":"2024-12-30T17:57:24.470051Z","shell.execute_reply":"2024-12-30T17:57:25.371079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter missing values for train and test datasets\ntrain_missing = train_missing_table[train_missing_table['Missing Values'] > 0].sort_values(by='Percentage (%)', ascending=False)\ntest_missing = test_missing_table[test_missing_table['Missing Values'] > 0].sort_values(by='Percentage (%)', ascending=False)\n\n# Set up the figure and subplots\nfig, axes = plt.subplots(1, 2, figsize=(14, 6), sharey=True)\n\n# Bar plot for train dataset\ntrain_colors = cm.get_cmap('viridis', len(train_missing_table))(range(len(train_missing_table)))\naxes[0].barh(train_missing.index, train_missing['Percentage (%)'], color=train_colors)\naxes[0].set_title('Percentage of Missing Values (Train Data)', fontsize=12)\naxes[0].set_xlabel('Percentage (%)', fontsize=10)\naxes[0].set_ylabel('Features', fontsize=10)\naxes[0].grid(axis='x', linestyle='--', alpha=0.6)\naxes[0].grid(axis='y', linestyle='--', alpha=0.6)\naxes[0].invert_yaxis() \n\n# Bar plot for test dataset\ntest_colors = cm.get_cmap('viridis', len(test_missing))(range(len(test_missing)))\naxes[1].barh(test_missing.index, test_missing['Percentage (%)'], color=test_colors)\naxes[1].set_title('Percentage of Missing Values (Test Data)', fontsize=12)\naxes[1].set_xlabel('Percentage (%)', fontsize=10)\naxes[1].grid(axis='x', linestyle='--', alpha=0.6)\naxes[1].grid(axis='y', linestyle='--', alpha=0.6)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:57:25.373423Z","iopub.execute_input":"2024-12-30T17:57:25.373801Z","iopub.status.idle":"2024-12-30T17:57:25.850388Z","shell.execute_reply.started":"2024-12-30T17:57:25.373766Z","shell.execute_reply":"2024-12-30T17:57:25.84954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter only features with missing values in the training dataset\nfeatures_with_missing = train_missing_table[train_missing_table['Missing Values'] > 0].index.tolist()\n\ndef analyze_nan_with_target_filtered(df, target_column, features):\n    missing_analysis = {}\n    \n    for col in features:\n        # Split the data into missing and non-missing subsets for the column\n        missing_mask = df[col].isnull()\n        non_missing_mask = ~missing_mask\n\n         # Calculate statistics for Premium Amount\n        stats = {\n            \"Missing Count\": missing_mask.sum(),\n            \"Non-Missing Count\": non_missing_mask.sum(),\n            \"Mean (Missing)\": df.loc[missing_mask, target_column].mean(),\n            \"Mean (Non-Missing)\": df.loc[non_missing_mask, target_column].mean(),\n            \"Median (Missing)\": df.loc[missing_mask, target_column].median(),\n            \"Median (Non-Missing)\": df.loc[non_missing_mask, target_column].median(),\n            \"Std Dev (Missing)\": df.loc[missing_mask, target_column].std(),\n            \"Std Dev (Non-Missing)\": df.loc[non_missing_mask, target_column].std(),\n        }\n        \n        missing_analysis[col] = stats\n    \n    return pd.DataFrame(missing_analysis).T\n\n# Perform the analysis for only features with missing values\nmissing_vs_premium_filtered = analyze_nan_with_target_filtered(train, \"Premium Amount\", features_with_missing)\n\n# Display the results\nprint(\"Analysis of Missing Values with Target (Premium Amount):\\n\")\ndisplay(missing_vs_premium_filtered)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:57:25.851261Z","iopub.execute_input":"2024-12-30T17:57:25.851608Z","iopub.status.idle":"2024-12-30T17:57:26.668494Z","shell.execute_reply.started":"2024-12-30T17:57:25.851574Z","shell.execute_reply":"2024-12-30T17:57:26.667608Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Imputation Strategies for Numeric and Categorical Data","metadata":{}},{"cell_type":"code","source":"# Filling Missing Values in Numeric Columns\nnumeric_columns = train.select_dtypes(include=['number']).columns\n\nfor col in numeric_columns:\n    if col in test.columns:\n        # Impute missing values with -1\n        train[col].fillna(-1, inplace=True)\n        test[col].fillna(-1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:57:26.669251Z","iopub.execute_input":"2024-12-30T17:57:26.669483Z","iopub.status.idle":"2024-12-30T17:57:26.753633Z","shell.execute_reply.started":"2024-12-30T17:57:26.669462Z","shell.execute_reply":"2024-12-30T17:57:26.752771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filling Missing Values in Object Columns\nobject_columns = train.select_dtypes(include=['object']).columns\nfor col in object_columns:\n    if col in test.columns:\n        train[col].fillna(\"Unknown\", inplace=True)\n        test[col].fillna(\"Unknown\", inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:57:26.754565Z","iopub.execute_input":"2024-12-30T17:57:26.754868Z","iopub.status.idle":"2024-12-30T17:57:27.802026Z","shell.execute_reply.started":"2024-12-30T17:57:26.754831Z","shell.execute_reply":"2024-12-30T17:57:27.801095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verify Missing Values\n\nprint(\"Missing Values After Imputation - Training Dataset:\")\nprint(train.isnull().sum())\n\nprint(\"\\nMissing Values After Imputation - Test Dataset:\")\nprint(test.isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:57:27.802824Z","iopub.execute_input":"2024-12-30T17:57:27.803071Z","iopub.status.idle":"2024-12-30T17:57:28.670858Z","shell.execute_reply.started":"2024-12-30T17:57:27.803051Z","shell.execute_reply":"2024-12-30T17:57:28.670093Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check for duplicate rows in the training dataset\ntrain_duplicates = train.duplicated().sum()\nprint(f\"\\nNumber of duplicate rows in the training dataset: {train_duplicates}\")\n\n# Check for duplicate rows in the test dataset\ntest_duplicates = test.duplicated().sum()\nprint(f\"Number of duplicate rows in the test dataset: {test_duplicates}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:57:28.671567Z","iopub.execute_input":"2024-12-30T17:57:28.671831Z","iopub.status.idle":"2024-12-30T17:57:30.855655Z","shell.execute_reply.started":"2024-12-30T17:57:28.67181Z","shell.execute_reply":"2024-12-30T17:57:30.85478Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"background-color: PINK; text-align: center; font-size: 24px; font-weight: bold; color: BLACK\">\nExploratory Data Analysis (EDA)\n</div>","metadata":{}},{"cell_type":"code","source":"# Target Column Extraction and Visualizing Distribution\n# Custom colormap using viridis\nviridis_cmap = cm.get_cmap(\"viridis\")\n\ndef visualize_premium_amount_with_peaks(data, feature='Premium Amount'):\n    plt.figure(figsize=(9, 4))\n\n    # Histogram with KDE\n    plt.subplot(1, 2, 1)\n    ax = sns.histplot(data[feature], bins=30, kde=True, color=viridis_cmap(0.5))\n    plt.title(f'Histogram of {feature} with KDE', fontsize=11)\n    plt.xlabel(feature, fontsize=10)\n    plt.ylabel('Frequency', fontsize=10)\n    plt.grid(True, linestyle='--', alpha=0.6) \n\n    # Extract KDE values to find peaks\n    kde = sns.kdeplot(data[feature], ax=ax, color=viridis_cmap(0.7)).lines[0].get_data()\n    kde_x, kde_y = kde[0], kde[1]\n\n\n    # Box Plot\n    plt.subplot(1, 2, 2)\n    sns.boxplot(x=data[feature], color=viridis_cmap(0.5))\n    plt.title(f'Box Plot of {feature}', fontsize=11)\n    plt.xlabel(feature, fontsize=10)\n    plt.grid(True, linestyle='--', alpha=0.6)  \n    plt.tight_layout()\n    plt.show()\n\nvisualize_premium_amount_with_peaks(train, feature='Premium Amount')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:57:30.856519Z","iopub.execute_input":"2024-12-30T17:57:30.856821Z","iopub.status.idle":"2024-12-30T17:57:40.515533Z","shell.execute_reply.started":"2024-12-30T17:57:30.856797Z","shell.execute_reply":"2024-12-30T17:57:40.514657Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Distribution Analysis of Numerical Features\n\n","metadata":{}},{"cell_type":"code","source":"# Define columns to analyze\ncolumns_to_analyze = train.select_dtypes(include=['number']).columns.drop('Premium Amount')\n\nviridis_cmap = cm.get_cmap(\"viridis\")\n# Extract three colors from the colormap\nviridis_colors = [viridis_cmap(0.3), viridis_cmap(0.5), viridis_cmap(0.8)]\n\nfig, axes = plt.subplots(len(columns_to_analyze), 3, figsize=(25, len(columns_to_analyze) * 5))\n\nfor i, column in enumerate(columns_to_analyze):\n    # Histogram for train_data\n    sns.histplot(train[column], bins=30, kde=True, color=viridis_colors[0], ax=axes[i, 0])\n    axes[i, 0].set_title(f'Distribution of {column} (Train)', fontsize=14)\n    axes[i, 0].set_xlabel(column, fontsize=10)\n    axes[i, 0].set_ylabel('Frequency', fontsize=10)\n    axes[i, 0].grid(visible=True, linestyle='--', alpha=0.6)\n\n    # Boxplot for train_data\n    sns.boxplot(x=train[column], color=viridis_colors[1], ax=axes[i, 1])\n    axes[i, 1].set_title(f'Boxplot of {column} (Train)', fontsize=14)\n    axes[i, 1].set_xlabel(column, fontsize=10)\n    axes[i, 1].grid(visible=True, linestyle='--', alpha=0.6)\n\n    # Boxplot for test_data\n    sns.boxplot(x=test[column], color=viridis_colors[2], ax=axes[i, 2])\n    axes[i, 2].set_title(f'Boxplot of {column} (Test)', fontsize=12)\n    axes[i, 2].set_xlabel(column, fontsize=10)\n    axes[i, 2].grid(visible=True, linestyle='--', alpha=0.6)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:57:40.516448Z","iopub.execute_input":"2024-12-30T17:57:40.516769Z","iopub.status.idle":"2024-12-30T17:58:19.599295Z","shell.execute_reply.started":"2024-12-30T17:57:40.516733Z","shell.execute_reply":"2024-12-30T17:58:19.598421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select numeric columns \nnumeric_data = train.select_dtypes(include=['number'])\n\n# Compute the correlation matrix\ncorrelation_matrix = numeric_data.corr()\nplt.figure(figsize=(8, 6))\n\n# Create the heatmap\nsns.heatmap(\n    correlation_matrix, \n    annot=True, \n    fmt=\".2f\", \n    cmap='viridis',  \n    cbar=True, \n    square=True,\n    mask=np.triu(np.ones_like(correlation_matrix, dtype=bool)),  \n    linewidths=0.5  \n)\n\nplt.title('Correlation Heatmap of Numerical Features (Excluding Target)', fontsize=12)\nplt.xticks(rotation=45, ha='right', fontsize=10)\nplt.yticks(fontsize=10)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:19.600194Z","iopub.execute_input":"2024-12-30T17:58:19.600436Z","iopub.status.idle":"2024-12-30T17:58:20.208093Z","shell.execute_reply.started":"2024-12-30T17:58:19.600415Z","shell.execute_reply":"2024-12-30T17:58:20.207289Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Categorical Feature Analysis","metadata":{}},{"cell_type":"code","source":"\n# Function to display barplot and pie chart for categorical columns\ndef plot_categorical_distribution(data, column_name):\n    plt.figure(figsize=(12, 4))\n    \n    # Bar plot for categorical distribution\n    plt.subplot(1, 2, 1)\n    sns.countplot(y=column_name, data=data, palette='Set2')\n    plt.title(f'Distribution of {column_name}', fontsize=12)\n    plt.xlabel('Count', fontsize=10)\n    plt.ylabel(column_name, fontsize=10)\n\n    ax = plt.gca()\n    for p in ax.patches:\n        count = int(p.get_width())\n        ax.annotate(f'{count}', \n                    (p.get_width() + 0.1, p.get_y() + p.get_height() / 2), \n                    ha='left', va='center', fontsize=10, color='black')\n    \n    sns.despine(left=True, bottom=True)\n    \n    # Pie chart for percentage distribution\n    plt.subplot(1, 2, 2)\n    data[column_name].value_counts().plot.pie(\n        autopct='%1.1f%%', \n        colors=sns.color_palette('Set2', data[column_name].nunique()), \n        startangle=90, \n        explode=[0.05] * data[column_name].nunique(), \n        shadow=True\n    )\n    plt.title(f'Percentage Distribution of {column_name}', fontsize=12)\n    plt.ylabel('')  \n\n    plt.tight_layout()\n    plt.show()\n\ncategorical_columns = ['Gender', 'Marital Status', 'Education Level', 'Occupation', 'Location', \n                'Policy Type', 'Customer Feedback', 'Smoking Status', 'Exercise Frequency', 'Property Type']\n\nfor column in categorical_columns:\n    plot_categorical_distribution(train, column)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:20.209119Z","iopub.execute_input":"2024-12-30T17:58:20.209416Z","iopub.status.idle":"2024-12-30T17:58:30.531466Z","shell.execute_reply.started":"2024-12-30T17:58:20.209383Z","shell.execute_reply":"2024-12-30T17:58:30.530623Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Categorical Features vs Premium Amount","metadata":{}},{"cell_type":"code","source":"# List of categorical columns\ncategorical_columns = [\n    'Gender', 'Marital Status', 'Education Level', 'Occupation', 'Location', \n    'Policy Type', 'Customer Feedback', 'Smoking Status', 'Exercise Frequency', 'Property Type'\n]\n\n# Loop through each categorical feature to display summary statistics and box plot\nfor column in categorical_columns:\n    # Calculate summary statistics grouped by the categorical column\n    stats = train.groupby(column)['Premium Amount'].agg(['mean', 'median', 'count'])\n    \n    # Display summary statistics\n    print(f\"\\nSummary Statistics for Premium Amount by {column}:\")\n    print(stats)\n    \n    # Plot box plot\n    plt.figure(figsize=(8, 4))\n    sns.boxplot(data=train, x=column, y='Premium Amount', palette='viridis')\n    plt.title(f'Premium Amount by {column}', fontsize=12)\n    plt.xlabel(column, fontsize=11)\n    plt.ylabel('Premium Amount', fontsize=11)\n    plt.xticks(rotation=45)\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:30.532229Z","iopub.execute_input":"2024-12-30T17:58:30.532447Z","iopub.status.idle":"2024-12-30T17:58:38.532129Z","shell.execute_reply.started":"2024-12-30T17:58:30.532417Z","shell.execute_reply":"2024-12-30T17:58:38.531369Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"background-color: PINK; text-align: center; font-size: 24px; font-weight: bold; color: BLACK\">\nDATA PREPROCESSING\n</div>","metadata":{}},{"cell_type":"markdown","source":"## Converting Date Columns to Epoch Time","metadata":{}},{"cell_type":"code","source":"# Retrieve columns with 'object' data type\ndatetime_columns = train.select_dtypes(include=['object']).columns\n\nfor col in datetime_columns:\n    try:\n        # Convert the column to datetime format\n        train[col] = pd.to_datetime(train[col], errors='raise')\n        test[col] = pd.to_datetime(test[col], errors='raise')\n        \n        # Convert datetime to epoch time (in seconds)\n        train[col] = train[col].astype(np.int64) / 10**9\n        test[col] = test[col].astype(np.int64) / 10**9\n\n        print(f\"Converted '{col}' to epoch time.\")\n    except Exception as e:\n        print(f\"Skipping column '{col}' due to: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:38.532676Z","iopub.execute_input":"2024-12-30T17:58:38.53292Z","iopub.status.idle":"2024-12-30T17:58:39.841785Z","shell.execute_reply.started":"2024-12-30T17:58:38.5329Z","shell.execute_reply":"2024-12-30T17:58:39.841009Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check the first few rows and data type of the 'Policy Start Date' column\ncolumn_name = 'Policy Start Date'\n\n# Display the first few rows\nprint(f\"Sample data for '{column_name}':\\n\", train[column_name].head())\n\n# Display the data type of the column\nprint(f\"Data type of '{column_name}' in train_data: {train[column_name].dtype}\")\n\n# Repeat for test_data\nprint(f\"Sample data for '{column_name}' in test_data:\\n\", test[column_name].head())\nprint(f\"Data type of '{column_name}' in test_data: {test[column_name].dtype}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:39.842567Z","iopub.execute_input":"2024-12-30T17:58:39.842891Z","iopub.status.idle":"2024-12-30T17:58:39.849373Z","shell.execute_reply.started":"2024-12-30T17:58:39.842862Z","shell.execute_reply":"2024-12-30T17:58:39.848622Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Label Encoding Categorical Features","metadata":{}},{"cell_type":"code","source":"def identify_non_numerical_features(data, dataset_name):\n    non_numerical_features = data.select_dtypes(include=['object'])\n    print(f\"Non-Numerical Features and Unique Values in {dataset_name} dataset:\")\n    for column in non_numerical_features.columns:\n        unique_values = non_numerical_features[column].unique()\n        print(f\"\\n{column}: {unique_values}\")\n\n# Apply the function to training and test datasets\nidentify_non_numerical_features(train, \"Training\")\nprint(\"\\n\")\nidentify_non_numerical_features(test, \"Test\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:39.850133Z","iopub.execute_input":"2024-12-30T17:58:39.850365Z","iopub.status.idle":"2024-12-30T17:58:41.447598Z","shell.execute_reply.started":"2024-12-30T17:58:39.850333Z","shell.execute_reply":"2024-12-30T17:58:41.446801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the encoding strategies for specific features\nbinary_features = ['Gender', 'Smoking Status']\nordinal_features = {\n    'Exercise Frequency': ['Rarely', 'Monthly', 'Weekly', 'Daily']\n}\nnominal_features = ['Marital Status', 'Education Level', 'Occupation', \n                    'Location', 'Policy Type', 'Customer Feedback', 'Property Type']\n\n# Binary Encoding for binary features\nle = LabelEncoder()\nfor feature in binary_features:\n    train[feature] = le.fit_transform(train[feature])\n    test[feature] = le.transform(test[feature])\n\n# Ordinal Encoding for ordered features\nfor feature, order in ordinal_features.items():\n    oe = OrdinalEncoder(categories=[order])\n    train[feature] = oe.fit_transform(train[[feature]]).flatten()  # Flatten to 1D\n    test[feature] = oe.transform(test[[feature]]).flatten()       # Flatten to 1D\n\n# One-Hot Encoding for nominal features\ntrain = pd.get_dummies(train, columns=nominal_features, drop_first=True)\ntest = pd.get_dummies(test, columns=nominal_features, drop_first=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:41.451335Z","iopub.execute_input":"2024-12-30T17:58:41.45155Z","iopub.status.idle":"2024-12-30T17:58:43.65157Z","shell.execute_reply.started":"2024-12-30T17:58:41.451531Z","shell.execute_reply":"2024-12-30T17:58:43.650911Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Verifying Data Types Across Datasets","metadata":{}},{"cell_type":"code","source":"# Create data type tables for train_data and test_data\ntrain_data_types = pd.DataFrame({\n    'Column Name': train.columns,\n    'Train Data Type': train.dtypes\n})\n\ntest_data_types = pd.DataFrame({\n    'Column Name': test.columns,\n    'Test Data Type': test.dtypes\n})\n\n# Merge the two tables for comparison\ndata_types_comparison = pd.merge(\n    train_data_types, \n    test_data_types, \n    on='Column Name', \n    how='outer'\n)\n\n# Display the data types comparison table\nprint(\"Data Types Comparison of Train and Test Datasets:\\n\")\ndisplay(data_types_comparison)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:43.652882Z","iopub.execute_input":"2024-12-30T17:58:43.653167Z","iopub.status.idle":"2024-12-30T17:58:43.672228Z","shell.execute_reply.started":"2024-12-30T17:58:43.653145Z","shell.execute_reply":"2024-12-30T17:58:43.671484Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Normalization of Numerical Features","metadata":{}},{"cell_type":"code","source":"target_column = (set(train.columns) - set(test.columns)).pop()\n\n# Select numerical columns\nnumerical_columns = train.select_dtypes(include=['float64']).columns\nnumerical_columns = numerical_columns[numerical_columns != target_column]\n\n# Applying Normalization\nscaler = StandardScaler()\ntrain[numerical_columns] = scaler.fit_transform(train[numerical_columns])\ntest[numerical_columns] = scaler.transform(test[numerical_columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:43.67293Z","iopub.execute_input":"2024-12-30T17:58:43.67312Z","iopub.status.idle":"2024-12-30T17:58:43.991723Z","shell.execute_reply.started":"2024-12-30T17:58:43.673103Z","shell.execute_reply":"2024-12-30T17:58:43.991057Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Distribution Analysis of Preprocessed Features","metadata":{}},{"cell_type":"code","source":"# Identify numeric columns only (excluding boolean columns)\nnumeric_columns = train.select_dtypes(include=[np.number]).columns\n\n# Calculate the number of rows and columns needed\nnum_features = len(numeric_columns)\nnum_cols = 4\nnum_rows = math.ceil(num_features / num_cols)\n\n# Create subplots\nfig, axes = plt.subplots(nrows=num_rows, ncols=num_cols, figsize=(16, num_rows * 3))\nviridis_cmap = cm.get_cmap('viridis', len(numeric_columns))\n\n# Plot each numeric column\nfor i, column in enumerate(numeric_columns):\n    ax = axes.flatten()[i]\n    train[column].hist(\n        ax=ax, \n        bins=20, \n        color=viridis_cmap(i / len(numeric_columns)),  \n        edgecolor='black', \n        linewidth=0.5\n    )\n    ax.set_title(column, fontsize=9)\n    ax.tick_params(axis='both', which='major', labelsize=6)\n    ax.grid(True, linestyle='--', alpha=0.6)  \n\n# Remove empty subplots if any\nfor j in range(i + 1, len(axes.flatten())):\n    fig.delaxes(axes.flatten()[j])\n\nplt.suptitle('Dataset Feature Distributions (train_data)', fontsize=11)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:43.992519Z","iopub.execute_input":"2024-12-30T17:58:43.992838Z","iopub.status.idle":"2024-12-30T17:58:46.937677Z","shell.execute_reply.started":"2024-12-30T17:58:43.992807Z","shell.execute_reply":"2024-12-30T17:58:46.936811Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Skewness Reduction with Log Transformation","metadata":{}},{"cell_type":"code","source":"# Define the continuous columns\ncontinuous_columns_train = ['Annual Income', 'Premium Amount']  \ncontinuous_columns_test = ['Annual Income']  \n\n# Calculate skewness for the specified continuous columns\ntrain_skewness = train[continuous_columns_train].apply(skew)\ntest_skewness = test[continuous_columns_test].apply(skew)\n\n# Display results\nprint(\"Skewness for Training Dataset:\\n\")\ndisplay(train_skewness)\n\nprint(\"\\nSkewness for Test Dataset:\\n\")\ndisplay(test_skewness)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:46.938762Z","iopub.execute_input":"2024-12-30T17:58:46.939072Z","iopub.status.idle":"2024-12-30T17:58:46.998599Z","shell.execute_reply.started":"2024-12-30T17:58:46.939044Z","shell.execute_reply":"2024-12-30T17:58:46.997904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Log-transform skewed features\ntrain['Annual Income'] = np.log1p(train['Annual Income'])\ntest['Annual Income'] = np.log1p(test['Annual Income'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:46.999311Z","iopub.execute_input":"2024-12-30T17:58:46.999528Z","iopub.status.idle":"2024-12-30T17:58:47.010457Z","shell.execute_reply.started":"2024-12-30T17:58:46.999504Z","shell.execute_reply":"2024-12-30T17:58:47.00968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select only numeric columns from the training data\nnumeric_data = train.select_dtypes(include=['number'])\n\n# Add a new column for the log-transformed values of target variable\nnumeric_data['Log_Transformed_Premium'] = np.log1p(train[target_column])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:47.011256Z","iopub.execute_input":"2024-12-30T17:58:47.01158Z","iopub.status.idle":"2024-12-30T17:58:47.150395Z","shell.execute_reply.started":"2024-12-30T17:58:47.011548Z","shell.execute_reply":"2024-12-30T17:58:47.149655Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Correlation Analysis of Preprocessed Data","metadata":{}},{"cell_type":"code","source":"# Compute the correlation matrix\ntrain_corr_matrix = numeric_data.corr()\n\n# Create the heatmap\nplt.figure(figsize=(12, 9))\nsns.heatmap(\n    train_corr_matrix, \n    annot=True, \n    cmap='viridis',   \n    vmax=1, \n    vmin=-1,\n    annot_kws={\"size\": 8}, \n    fmt=\".3f\",\n    linewidths=0.5  \n)\n\n# Customize x and y tick labels\nplt.xticks(rotation=80, fontsize=9)\nplt.yticks(fontsize=9)\n\n# Add title and layout adjustments\nplt.title(\"Correlation Heatmap of the Train Data\", fontsize=11)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:47.151135Z","iopub.execute_input":"2024-12-30T17:58:47.151366Z","iopub.status.idle":"2024-12-30T17:58:48.624343Z","shell.execute_reply.started":"2024-12-30T17:58:47.151347Z","shell.execute_reply":"2024-12-30T17:58:48.623543Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Separating Features and Target","metadata":{}},{"cell_type":"code","source":"# Separate Features and Target\nX_train = train.drop([target_column], axis=1)  # Features\ny_train = train[target_column]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:48.625331Z","iopub.execute_input":"2024-12-30T17:58:48.625664Z","iopub.status.idle":"2024-12-30T17:58:48.699354Z","shell.execute_reply.started":"2024-12-30T17:58:48.625628Z","shell.execute_reply":"2024-12-30T17:58:48.69859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying Log Transformation to the target variable\ny_train_log = np.log1p(y_train)  # log1p is used for log(1 + x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:48.700138Z","iopub.execute_input":"2024-12-30T17:58:48.700442Z","iopub.status.idle":"2024-12-30T17:58:48.710396Z","shell.execute_reply.started":"2024-12-30T17:58:48.700412Z","shell.execute_reply":"2024-12-30T17:58:48.709468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Custom colormap using viridis\nviridis_cmap = cm.get_cmap(\"viridis\")\n\n# Select two colors from the colormap\ncolor1 = viridis_cmap(0.5)  \ncolor2 = viridis_cmap(0.3)  \n\n# Plot with the custom colors\nplt.figure(figsize=(9, 4))\n\n# Plot original target distribution\nplt.subplot(1, 2, 1)\nsns.histplot(y_train, kde=True, bins=30, color=color1)\nplt.title(f'Histogram of Target: {target_column} (y)', fontsize=11)\nplt.xlabel(f'{target_column} (y)', fontsize=10)\nplt.ylabel('Frequency', fontsize=10)\nplt.tick_params(axis='both', which='major', labelsize=7)\nplt.grid(True, linestyle='--', alpha=0.6)\n\n# Log-transformed target distribution\nplt.subplot(1, 2, 2)\nsns.histplot(y_train_log, kde=True, bins=30, color=color2)\nplt.title('Histogram of log(y + 1)', fontsize=11)\nplt.xlabel('log(y + 1)', fontsize=10)\nplt.ylabel('Frequency', fontsize=10)\nplt.tick_params(axis='both', which='major', labelsize=7)\nplt.grid(True, linestyle='--', alpha=0.6)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:48.7113Z","iopub.execute_input":"2024-12-30T17:58:48.71159Z","iopub.status.idle":"2024-12-30T17:58:58.15423Z","shell.execute_reply.started":"2024-12-30T17:58:48.711555Z","shell.execute_reply":"2024-12-30T17:58:58.153376Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Removing Whitespaces in Feature Names","metadata":{}},{"cell_type":"code","source":"# Remove whitespaces in feature names for X_train and test_data\nX_train.columns = X_train.columns.str.replace(' ', '_', regex=True)\ntest.columns = test.columns.str.replace(' ', '_', regex=True)\n\n# Verify the updated column names\nprint(\"Updated Feature Names in X_train:\")\nprint(X_train.columns)\n\nprint(\"\\nUpdated Feature Names in test_data:\")\nprint(test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:58.155116Z","iopub.execute_input":"2024-12-30T17:58:58.155442Z","iopub.status.idle":"2024-12-30T17:58:58.162066Z","shell.execute_reply.started":"2024-12-30T17:58:58.15541Z","shell.execute_reply":"2024-12-30T17:58:58.161295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display data types\nprint(\"Feature Data Types (X_train):\")\nprint(X_train.dtypes)\n\nprint(\"\\nTarget Data Type (y_train):\")\nprint(y_train.dtypes)\n\nprint(\"\\nLog-Transformed Target Data Type (y_train_log):\")\nprint(y_train_log.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:58.162719Z","iopub.execute_input":"2024-12-30T17:58:58.162944Z","iopub.status.idle":"2024-12-30T17:58:58.183826Z","shell.execute_reply.started":"2024-12-30T17:58:58.162923Z","shell.execute_reply":"2024-12-30T17:58:58.183041Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"background-color: PINK; text-align: center; font-size: 24px; font-weight: bold; color: BLACK\">\nMODEL TRAINING\n</div>","metadata":{}},{"cell_type":"markdown","source":"## Defining the Objective Function for Hyperparameter Optimization\nSetting Up an Objective Function for Optuna\n\nHyperparameter Search Space Definition for LightGBM","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n    # Define the search space\n    param = {\n        \"objective\": \"regression\",\n        \"metric\": \"rmse\",\n        \"boosting_type\": \"gbdt\",  \n        \"device\": \"gpu\",  \n        \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 0.01, 0.2),\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 100, 1000),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 3, 15),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 300),\n        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 10, 100),\n        \"subsample\": trial.suggest_uniform(\"subsample\", 0.5, 1.0),\n        \"colsample_bytree\": trial.suggest_uniform(\"colsample_bytree\", 0.5, 1.0),\n        \"reg_alpha\": trial.suggest_loguniform(\"reg_alpha\", 1e-3, 10.0),\n        \"reg_lambda\": trial.suggest_loguniform(\"reg_lambda\", 1e-3, 10.0),\n        \"verbose\": -1  # Suppress warnings and messages\n    }\n    \n    # Train/validation split\n    X_train_split, X_val, y_train_split, y_val = train_test_split(X_train, y_train_log, test_size=0.2, random_state=42)\n    \n    # Train the model with callbacks\n    model = lgb.LGBMRegressor(**param)\n    model.fit(\n        X_train_split, y_train_split,\n        eval_set=[(X_val, y_val)],\n        callbacks=[early_stopping(stopping_rounds=30, verbose=False), log_evaluation(10)]\n    )\n    \n    # Predict and compute RMSLE\n    y_pred = model.predict(X_val)\n    rmsle = mean_squared_log_error(y_val, y_pred) ** 0.5\n    return rmsle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:58.184604Z","iopub.execute_input":"2024-12-30T17:58:58.184879Z","iopub.status.idle":"2024-12-30T17:58:58.195866Z","shell.execute_reply.started":"2024-12-30T17:58:58.184858Z","shell.execute_reply":"2024-12-30T17:58:58.195184Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Optimizing Hyperparameters with Optuna\nConducting Hyperparameter Tuning with Optuna\n\nBest Parameter Search Using Optuna","metadata":{}},{"cell_type":"code","source":"# Create and optimize the study\nstudy = optuna.create_study(direction=\"minimize\")\nstudy.optimize(objective, n_trials=25)\n\n# Display the best parameters and score\nprint(\"Best parameters:\", study.best_params)\nprint(\"Best RMSLE:\", study.best_value)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:58:58.196489Z","iopub.execute_input":"2024-12-30T17:58:58.196781Z","iopub.status.idle":"2024-12-30T18:05:59.988769Z","shell.execute_reply.started":"2024-12-30T17:58:58.196759Z","shell.execute_reply":"2024-12-30T18:05:59.987928Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training the Final LightGBM Model\nModel Training with Optimized Parameters\n\nFinal Model Fitting with Best Hyperparameters","metadata":{}},{"cell_type":"code","source":"# Extract best parameters\nbest_params = study.best_params\nbest_params[\"objective\"] = \"regression\"\nbest_params[\"metric\"] = \"rmse\"\nbest_params[\"device\"] = \"gpu\"  # Ensure GPU is used for the final model\n\n# Train the final model\nfinal_model = lgb.LGBMRegressor(**best_params)\nfinal_model.fit(X_train, y_train_log)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T18:05:59.989628Z","iopub.execute_input":"2024-12-30T18:05:59.989907Z","iopub.status.idle":"2024-12-30T18:06:23.794709Z","shell.execute_reply.started":"2024-12-30T18:05:59.989883Z","shell.execute_reply":"2024-12-30T18:06:23.793855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict on training data\ny_train_pred = final_model.predict(X_train)\ntrain_rmsle = mean_squared_log_error(y_train_log, y_train_pred) ** 0.5\nprint(\"RMSLE on Training Data:\", train_rmsle)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T18:06:23.795446Z","iopub.execute_input":"2024-12-30T18:06:23.795665Z","iopub.status.idle":"2024-12-30T18:07:05.107054Z","shell.execute_reply.started":"2024-12-30T18:06:23.795645Z","shell.execute_reply":"2024-12-30T18:07:05.106273Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Making Predictions on Test Data\nGenerating Predictions for the Test Set\n\nTest Data Predictions Using the Final LightGBM Model","metadata":{}},{"cell_type":"code","source":"# Predict on test data\ny_test_pred = np.expm1(final_model.predict(test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T18:07:05.10778Z","iopub.execute_input":"2024-12-30T18:07:05.107987Z","iopub.status.idle":"2024-12-30T18:07:32.671852Z","shell.execute_reply.started":"2024-12-30T18:07:05.107969Z","shell.execute_reply":"2024-12-30T18:07:32.670994Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Importance Analysis\nIdentifying Key Features Using Importance Scores\n\nFeature Importance Visualization for LightGBM Model","metadata":{}},{"cell_type":"code","source":"# Extract feature importance from the trained LightGBM model\nfeature_importance = pd.DataFrame({\n    'Feature': X_train.columns,\n    'Importance': final_model.feature_importances_\n})\n\n# Sort features by importance\nfeature_importance = feature_importance.sort_values(by='Importance', ascending=False)\n\n# Plot the feature importance\nplt.figure(figsize=(10, 6))\nsns.barplot(\n    x='Importance', \n    y='Feature', \n    data=feature_importance,\n    palette='viridis'\n)\nplt.title('Feature Importance - LightGBM Model', fontsize=14)\nplt.xlabel('Importance Score', fontsize=12)\nplt.ylabel('Features', fontsize=12)\nplt.grid(axis='x', linestyle='--', alpha=0.6)\nplt.grid(axis='y', linestyle='--', alpha=0.6)\nplt.tight_layout()\nplt.show()\n\n# Display top 10 important features\nprint(\"Top 10 Features by Importance:\\n\")\ndisplay(feature_importance.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T18:07:32.672767Z","iopub.execute_input":"2024-12-30T18:07:32.673047Z","iopub.status.idle":"2024-12-30T18:07:33.13637Z","shell.execute_reply.started":"2024-12-30T18:07:32.673023Z","shell.execute_reply":"2024-12-30T18:07:33.135477Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Analyzing Prediction Distributions\nComparing True and Predicted Distributions\n\nDistribution Insights for Train and Test Predictions","metadata":{}},{"cell_type":"code","source":"# Visualization of Prediction Distributions\nviridis_cmap = cm.get_cmap(\"viridis\", 3) \n\nplt.figure(figsize=(10, 4))\n\n# Plot true values (Train Data)\nplt.hist(\n    y_train_log, bins=30, color=viridis_cmap(0), alpha=0.6, edgecolor=\"black\", label=\"True Values - Train\"\n)\n\n# Plot predicted values (Train Data)\nplt.hist(\n    y_train_pred, bins=30, color=viridis_cmap(0.5), alpha=0.6, edgecolor=\"black\", label=\"Predicted Values - Train\"\n)\n\n# Plot predicted values (Test Data)\nplt.hist(\n    np.log1p(y_test_pred), bins=30, color=viridis_cmap(0.8), alpha=0.6, edgecolor=\"black\", label=\"Predicted Values - Test\"\n)\n\n# Add titles and labels\nplt.title(\"Prediction Distributions (Train and Test Data)\", fontsize=14)\nplt.xlabel(\"Log-transformed Premium Amount (log(y + 1))\", fontsize=12)\nplt.ylabel(\"Frequency\", fontsize=12)\nplt.legend(fontsize=10)\nplt.grid(True, linestyle=\"--\", alpha=0.6)\n\n# Display plot\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T18:07:33.137472Z","iopub.execute_input":"2024-12-30T18:07:33.137856Z","iopub.status.idle":"2024-12-30T18:07:33.532529Z","shell.execute_reply.started":"2024-12-30T18:07:33.137822Z","shell.execute_reply":"2024-12-30T18:07:33.531777Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"background-color: PINK; text-align: center; font-size: 24px; font-weight: bold; color: BLACK\">\nCreating the Submission File\n</div>","metadata":{}},{"cell_type":"code","source":"# Prepare submission\nsample_submission = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\nsample_submission[\"Premium Amount\"] = y_test_pred\n\nprint(sample_submission.head(10))\n\nsample_submission.to_csv('submission.csv', index=False)\nsample_submission.head()\nprint(\"Submission file created successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T18:07:33.533281Z","iopub.execute_input":"2024-12-30T18:07:33.53357Z","iopub.status.idle":"2024-12-30T18:07:35.094475Z","shell.execute_reply.started":"2024-12-30T18:07:33.533546Z","shell.execute_reply":"2024-12-30T18:07:35.093768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}