{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:27:10.607758Z","iopub.execute_input":"2024-12-04T17:27:10.608336Z","iopub.status.idle":"2024-12-04T17:27:11.945739Z","shell.execute_reply.started":"2024-12-04T17:27:10.608296Z","shell.execute_reply":"2024-12-04T17:27:11.944439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:27:27.761857Z","iopub.execute_input":"2024-12-04T17:27:27.762453Z","iopub.status.idle":"2024-12-04T17:27:38.984529Z","shell.execute_reply.started":"2024-12-04T17:27:27.762408Z","shell.execute_reply":"2024-12-04T17:27:38.983233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:28:02.718114Z","iopub.execute_input":"2024-12-04T17:28:02.718552Z","iopub.status.idle":"2024-12-04T17:28:02.766082Z","shell.execute_reply.started":"2024-12-04T17:28:02.718515Z","shell.execute_reply":"2024-12-04T17:28:02.76485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"The shape of train data: \", train_df.shape)\nprint(\"The shape of test data: \", test_df.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:29:34.419557Z","iopub.execute_input":"2024-12-04T17:29:34.420003Z","iopub.status.idle":"2024-12-04T17:29:34.426312Z","shell.execute_reply.started":"2024-12-04T17:29:34.419964Z","shell.execute_reply":"2024-12-04T17:29:34.424885Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:31:53.24771Z","iopub.execute_input":"2024-12-04T17:31:53.24813Z","iopub.status.idle":"2024-12-04T17:31:54.006178Z","shell.execute_reply.started":"2024-12-04T17:31:53.248092Z","shell.execute_reply":"2024-12-04T17:31:54.005021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:05:04.533313Z","iopub.execute_input":"2024-12-04T19:05:04.533766Z","iopub.status.idle":"2024-12-04T19:05:05.268195Z","shell.execute_reply.started":"2024-12-04T19:05:04.53373Z","shell.execute_reply":"2024-12-04T19:05:05.266916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(\"The missing values: \\n\")\nnull_counts = train_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:24:25.182962Z","iopub.execute_input":"2024-12-04T19:24:25.183388Z","iopub.status.idle":"2024-12-04T19:24:25.83904Z","shell.execute_reply.started":"2024-12-04T19:24:25.183348Z","shell.execute_reply":"2024-12-04T19:24:25.837872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"null_counts = null_counts[null_counts > 0]\n\n# Plot the bar chart\nplt.figure(figsize=(10, 5))\nnull_counts.plot(kind='bar', color='skyblue')\nplt.title(\"Count of Null Values per Column\")\nplt.xlabel(\"Columns\")\nplt.ylabel(\"Null Value Count\")\nplt.xticks(rotation=75)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:24:26.688317Z","iopub.execute_input":"2024-12-04T19:24:26.688705Z","iopub.status.idle":"2024-12-04T19:24:27.621007Z","shell.execute_reply.started":"2024-12-04T19:24:26.688671Z","shell.execute_reply":"2024-12-04T19:24:27.619562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_percentage = train_df.isnull().mean() * 100\ncols_to_remove = missing_percentage[missing_percentage > 30].index.tolist()\nprint(cols_to_remove)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:05:30.760655Z","iopub.execute_input":"2024-12-04T19:05:30.76108Z","iopub.status.idle":"2024-12-04T19:05:31.419019Z","shell.execute_reply.started":"2024-12-04T19:05:30.761041Z","shell.execute_reply":"2024-12-04T19:05:31.417843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.drop(columns=['Previous Claims', 'id'], inplace=True)\ntest_df.drop(columns=['Previous Claims'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:44:07.402479Z","iopub.execute_input":"2024-12-04T17:44:07.402959Z","iopub.status.idle":"2024-12-04T17:44:07.903754Z","shell.execute_reply.started":"2024-12-04T17:44:07.402918Z","shell.execute_reply":"2024-12-04T17:44:07.902285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get numerical columns\nnumerical_cols = train_df.select_dtypes(include=['int64', 'float64']).columns.tolist()\n\n# Get categorical columns\ncategorical_cols = train_df.select_dtypes(include=['object', 'category']).columns.tolist()\n\nprint(\"Numerical Columns:\", numerical_cols)\nprint(\"Categorical Columns:\", categorical_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:18:04.546101Z","iopub.execute_input":"2024-12-04T18:18:04.546813Z","iopub.status.idle":"2024-12-04T18:18:05.114465Z","shell.execute_reply.started":"2024-12-04T18:18:04.546766Z","shell.execute_reply":"2024-12-04T18:18:05.113124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get numerical columns for test data\nnumerical_colsT = test_df.select_dtypes(include=['int64', 'float64']).columns.tolist()\n\n# Get categorical columns for test data\ncategorical_colsT = test_df.select_dtypes(include=['object', 'category']).columns.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:03:41.995863Z","iopub.execute_input":"2024-12-04T19:03:41.996329Z","iopub.status.idle":"2024-12-04T19:03:42.379652Z","shell.execute_reply.started":"2024-12-04T19:03:41.996288Z","shell.execute_reply":"2024-12-04T19:03:42.378478Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate a color palette\npalette = sns.color_palette(\"husl\", len(numerical_cols))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:21:58.178282Z","iopub.execute_input":"2024-12-04T18:21:58.178872Z","iopub.status.idle":"2024-12-04T18:21:58.185882Z","shell.execute_reply.started":"2024-12-04T18:21:58.178752Z","shell.execute_reply":"2024-12-04T18:21:58.184675Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create boxplots\nplt.figure(figsize=(10, 6))\nfor i, (col, color) in enumerate(zip(numerical_cols, palette), 1):\n    plt.subplot(4, 2, i)  # Adjust rows and columns based on number of plots\n    sns.boxplot(data=train_df, x=col, color=color)\n    plt.title(f'Boxplot of {col}')\n    plt.xlabel('')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:33:45.303332Z","iopub.execute_input":"2024-12-04T18:33:45.303806Z","iopub.status.idle":"2024-12-04T18:33:46.96962Z","shell.execute_reply.started":"2024-12-04T18:33:45.303762Z","shell.execute_reply":"2024-12-04T18:33:46.967878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nfor i, col in enumerate(numerical_cols, 1):\n    plt.subplot(4, 2, i)  # Adjust rows and columns based on number of plots\n    sns.histplot(train_df[col], kde=True, bins=30, color= palette[i-1])\n    plt.title(f'Distribution of {col}')\n    plt.xlabel(col)\n    plt.ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:34:07.099901Z","iopub.execute_input":"2024-12-04T18:34:07.100353Z","iopub.status.idle":"2024-12-04T18:34:49.809985Z","shell.execute_reply.started":"2024-12-04T18:34:07.100311Z","shell.execute_reply":"2024-12-04T18:34:49.808704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to replace outliers with median\ndef replace_outliers_with_median(df, cols):\n    for col in cols:\n        # Calculate Q1, Q3, and IQR\n        Q1 = df[col].quantile(0.25)\n        Q3 = df[col].quantile(0.75)\n        IQR = Q3 - Q1\n\n        # Define the lower and upper bounds for outliers\n        lower_bound = Q1 - 1.5 * IQR\n        upper_bound = Q3 + 1.5 * IQR\n\n        # Replace outliers with the median\n        median = df[col].median()\n        df[col] = df[col].apply(lambda x: median if x < lower_bound or x > upper_bound else x)\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:50:46.184688Z","iopub.execute_input":"2024-12-04T18:50:46.185184Z","iopub.status.idle":"2024-12-04T18:50:46.192605Z","shell.execute_reply.started":"2024-12-04T18:50:46.185122Z","shell.execute_reply":"2024-12-04T18:50:46.191453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply the function to replace outliers with median\ntrain_df = replace_outliers_with_median(train_df, numerical_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:51:05.661359Z","iopub.execute_input":"2024-12-04T18:51:05.661796Z","iopub.status.idle":"2024-12-04T18:51:09.496324Z","shell.execute_reply.started":"2024-12-04T18:51:05.661748Z","shell.execute_reply":"2024-12-04T18:51:09.49492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply the function to replace outliers with median\ntest_df = replace_outliers_with_median(test_df, numerical_colsT)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def replaceMissingValues(df,cols):\n    for col in cols:\n        # Replace null values with the median\n        mean = df[col].mean()\n        df[col] = df[col].fillna(mean)\n        \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:03:18.884477Z","iopub.execute_input":"2024-12-04T19:03:18.88494Z","iopub.status.idle":"2024-12-04T19:03:18.892134Z","shell.execute_reply.started":"2024-12-04T19:03:18.884901Z","shell.execute_reply":"2024-12-04T19:03:18.890339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = replaceMissingValues(train_df, numerical_cols)\ntest_df = replaceMissingValues(test_df, numerical_colsT)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:03:50.557049Z","iopub.execute_input":"2024-12-04T19:03:50.557495Z","iopub.status.idle":"2024-12-04T19:03:50.717142Z","shell.execute_reply.started":"2024-12-04T19:03:50.557455Z","shell.execute_reply":"2024-12-04T19:03:50.715909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for cols in categorical_cols:\n    print(f\"{cols} unique values are: \", train_df[cols].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:14:58.511926Z","iopub.execute_input":"2024-12-04T19:14:58.512342Z","iopub.status.idle":"2024-12-04T19:14:59.479459Z","shell.execute_reply.started":"2024-12-04T19:14:58.512304Z","shell.execute_reply":"2024-12-04T19:14:59.47823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply forward fill to replace nulls in categorical columns\ntrain_df[categorical_cols] = train_df[categorical_cols].fillna(method='ffill')\ntest_df[categorical_colsT] = test_df[categorical_colsT].fillna(method='ffill')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:23:48.175243Z","iopub.execute_input":"2024-12-04T19:23:48.175708Z","iopub.status.idle":"2024-12-04T19:23:51.151781Z","shell.execute_reply.started":"2024-12-04T19:23:48.175659Z","shell.execute_reply":"2024-12-04T19:23:51.150513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(25, 20))\nfor i, col in enumerate(numerical_cols[:-1], 1):  # excluding 'Premium Amount'\n    plt.subplot(3, 3, i)  # Adjust rows and columns based on number of plots\n    sns.histplot(x=train_df[col], y=train_df['Premium Amount'], color=palette[i-1])\n    plt.title(f'{col} vs Premium Amount')\n    plt.xlabel(col)\n    plt.ylabel('Premium Amount')\n\n# plt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:40:32.242408Z","iopub.execute_input":"2024-12-04T19:40:32.242868Z","iopub.status.idle":"2024-12-04T19:40:41.330243Z","shell.execute_reply.started":"2024-12-04T19:40:32.242825Z","shell.execute_reply":"2024-12-04T19:40:41.329048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for column in categorical_cols:\n    plt.figure(figsize=(8, 4))\n    sns.countplot(data=train_df, x=column, order=train_df[column].value_counts().index, palette=\"viridis\")\n    plt.title(f\"Frequency Distribution of {column}\")\n    plt.xticks(rotation=45)\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:51:48.075526Z","iopub.execute_input":"2024-12-04T19:51:48.075957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}