{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:29.069677Z","iopub.execute_input":"2026-01-07T12:38:29.069984Z","iopub.status.idle":"2026-01-07T12:38:29.079975Z","shell.execute_reply.started":"2026-01-07T12:38:29.069948Z","shell.execute_reply":"2026-01-07T12:38:29.078632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as up #  Importing necessary libraries for data manipulation (pandas, numpy)\nimport pandas as pd \nimport warnings # visualization (matplotlib, seaborn), and to ignore warnings\nimport plotly.express as px \nimport matplotlib.pyplot as plt\nimport seaborn as sns ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:29.081993Z","iopub.execute_input":"2026-01-07T12:38:29.082261Z","iopub.status.idle":"2026-01-07T12:38:29.111586Z","shell.execute_reply.started":"2026-01-07T12:38:29.082239Z","shell.execute_reply":"2026-01-07T12:38:29.10998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"warnings.filterwarnings('ignore') # visualization to ignore warnings","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:29.113812Z","iopub.execute_input":"2026-01-07T12:38:29.11424Z","iopub.status.idle":"2026-01-07T12:38:29.135602Z","shell.execute_reply.started":"2026-01-07T12:38:29.114199Z","shell.execute_reply":"2026-01-07T12:38:29.134372Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 1 - Reading the Data\n\ntasks\nCalculate the percentaage of missing values ?\n\nHandle data types\n\nPlotting the data ( EDA ) to explore potential handling techniques\n\nDetect the outlier\n\nExplore How to visulaize the Date and time data in column ( Policy Start Date )","metadata":{}},{"cell_type":"markdown","source":" STEP 1:  Data Loading & Initial Inspection","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ndf # Loading the insurance dataset into a DataFrame","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:29.136593Z","iopub.execute_input":"2026-01-07T12:38:29.136849Z","iopub.status.idle":"2026-01-07T12:38:35.039616Z","shell.execute_reply.started":"2026-01-07T12:38:29.136828Z","shell.execute_reply":"2026-01-07T12:38:35.038488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head() # Displaying the first 5 rows of the data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:35.041938Z","iopub.execute_input":"2026-01-07T12:38:35.042842Z","iopub.status.idle":"2026-01-07T12:38:35.067241Z","shell.execute_reply.started":"2026-01-07T12:38:35.042811Z","shell.execute_reply":"2026-01-07T12:38:35.065396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.info # Checking for null values and data types","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:35.068855Z","iopub.execute_input":"2026-01-07T12:38:35.069448Z","iopub.status.idle":"2026-01-07T12:38:35.508115Z","shell.execute_reply.started":"2026-01-07T12:38:35.069419Z","shell.execute_reply":"2026-01-07T12:38:35.507223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.shape # Get the dimensions (rows, columns) of the DataFrame","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:35.509484Z","iopub.execute_input":"2026-01-07T12:38:35.509875Z","iopub.status.idle":"2026-01-07T12:38:35.51832Z","shell.execute_reply.started":"2026-01-07T12:38:35.509841Z","shell.execute_reply":"2026-01-07T12:38:35.51739Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"STEP 2 : DATA QUALITY ASSESSMENT","metadata":{}},{"cell_type":"code","source":"df.isnull().sum() # Count the number of missing (null) values in each column","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:35.519676Z","iopub.execute_input":"2026-01-07T12:38:35.519975Z","iopub.status.idle":"2026-01-07T12:38:36.380897Z","shell.execute_reply.started":"2026-01-07T12:38:35.519935Z","shell.execute_reply":"2026-01-07T12:38:36.379691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in df.columns: # Calculate and display the percentage of missing values for each column\n    col_nun = df[i].isnull().sum()\n    na_per = (col_nun / df.shape[0]) * 100\n    print(f'Missing in {i} is : {na_per.round(4)}%')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:36.382371Z","iopub.execute_input":"2026-01-07T12:38:36.382763Z","iopub.status.idle":"2026-01-07T12:38:37.066031Z","shell.execute_reply.started":"2026-01-07T12:38:36.382727Z","shell.execute_reply":"2026-01-07T12:38:37.065052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.duplicated().sum() # Count the number of duplicate rows in the DataFrame.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:37.069187Z","iopub.execute_input":"2026-01-07T12:38:37.069659Z","iopub.status.idle":"2026-01-07T12:38:38.89609Z","shell.execute_reply.started":"2026-01-07T12:38:37.06963Z","shell.execute_reply":"2026-01-07T12:38:38.895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.drop(columns=['id'], inplace=True) # Remove the id column because it's not useful for prediction","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:38.897216Z","iopub.execute_input":"2026-01-07T12:38:38.898114Z","iopub.status.idle":"2026-01-07T12:38:39.116246Z","shell.execute_reply.started":"2026-01-07T12:38:38.898071Z","shell.execute_reply":"2026-01-07T12:38:39.115248Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"STEP 3: HANDLE DATA TYPES","metadata":{}},{"cell_type":"code","source":"df['Policy Start Date'] = df['Policy Start Date'].astype('datetime64[ns]') # Convert the Policy Start Date column from text to datetime format.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:55:34.110445Z","iopub.execute_input":"2026-01-07T12:55:34.110834Z","iopub.status.idle":"2026-01-07T12:55:34.119694Z","shell.execute_reply.started":"2026-01-07T12:55:34.110807Z","shell.execute_reply":"2026-01-07T12:55:34.117775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat = df.select_dtypes('object').columns # Select all columns with text (object) data type and store them in variable cat.\ncat","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:39.749383Z","iopub.execute_input":"2026-01-07T12:38:39.749885Z","iopub.status.idle":"2026-01-07T12:38:40.283389Z","shell.execute_reply.started":"2026-01-07T12:38:39.74983Z","shell.execute_reply":"2026-01-07T12:38:40.282346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in cat: # Convert all text columns to category type to reduce memory usage.\n    df[i] = df[i].astype('category')\ndf.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:40.284555Z","iopub.execute_input":"2026-01-07T12:38:40.28495Z","iopub.status.idle":"2026-01-07T12:38:41.255044Z","shell.execute_reply.started":"2026-01-07T12:38:40.284924Z","shell.execute_reply":"2026-01-07T12:38:41.25398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num = df.select_dtypes(['float64', 'int64']).columns # Select all columns with numerical data types and store them in variable num.\nnum","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:41.256263Z","iopub.execute_input":"2026-01-07T12:38:41.25672Z","iopub.status.idle":"2026-01-07T12:38:41.299232Z","shell.execute_reply.started":"2026-01-07T12:38:41.256694Z","shell.execute_reply":"2026-01-07T12:38:41.297906Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"STEP 4: HANDLE MISSING VALUES","metadata":{}},{"cell_type":"code","source":"for col in cat: # Fill missing values in categorical columns with the mode (most frequent value).\n    if df[col].isnull().sum() > 0:\n        df[col].fillna(df[col].mode()[0], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:41.300583Z","iopub.execute_input":"2026-01-07T12:38:41.300958Z","iopub.status.idle":"2026-01-07T12:38:41.365538Z","shell.execute_reply.started":"2026-01-07T12:38:41.30092Z","shell.execute_reply":"2026-01-07T12:38:41.364408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in num: # Fill missing values in numerical columns with the median (middle value).\n    if df[col].isnull().sum() > 0:\n        df[col].fillna(df[col].median(), inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:41.366751Z","iopub.execute_input":"2026-01-07T12:38:41.367298Z","iopub.status.idle":"2026-01-07T12:38:41.613359Z","shell.execute_reply.started":"2026-01-07T12:38:41.367256Z","shell.execute_reply":"2026-01-07T12:38:41.612363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isnull().sum().sum() # Verify that all missing values have been handled by counting total nulls.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:41.614681Z","iopub.execute_input":"2026-01-07T12:38:41.615649Z","iopub.status.idle":"2026-01-07T12:38:41.667243Z","shell.execute_reply.started":"2026-01-07T12:38:41.615618Z","shell.execute_reply":"2026-01-07T12:38:41.665979Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"STEP 5: EXPLORATORY DATA ANALYSIS (EDA)","metadata":{}},{"cell_type":"code","source":"df[num].hist(bins=30, figsize=(15, 10), layout=(3, 3)) # Create histograms for all numerical columns to see their distributions\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:41.668409Z","iopub.execute_input":"2026-01-07T12:38:41.668809Z","iopub.status.idle":"2026-01-07T12:38:43.802749Z","shell.execute_reply.started":"2026-01-07T12:38:41.668772Z","shell.execute_reply":"2026-01-07T12:38:43.801438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6)) # Create a detailed histogram of the target variable Premium Amount with a density curve.\nsns.histplot(df['Premium Amount'], kde=True, bins=50)\nplt.title('Distribution of Premium Amount')\nplt.show()\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:43.804237Z","iopub.execute_input":"2026-01-07T12:38:43.804668Z","iopub.status.idle":"2026-01-07T12:38:49.759833Z","shell.execute_reply.started":"2026-01-07T12:38:43.804637Z","shell.execute_reply":"2026-01-07T12:38:49.758644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.boxplot(x=df['Premium Amount'])\nplt.title('Box Plot of Premium Amount')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:49.761483Z","iopub.execute_input":"2026-01-07T12:38:49.761871Z","iopub.status.idle":"2026-01-07T12:38:51.814006Z","shell.execute_reply.started":"2026-01-07T12:38:49.761833Z","shell.execute_reply":"2026-01-07T12:38:51.812783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6)) # Compare premium amounts between smokers and non-smokers using a box plot.\nsns.boxplot(x='Smoking Status', y='Premium Amount', data=df)\nplt.title('Premium Amount by Smoking Status')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:51.815385Z","iopub.execute_input":"2026-01-07T12:38:51.816197Z","iopub.status.idle":"2026-01-07T12:38:53.477184Z","shell.execute_reply.started":"2026-01-07T12:38:51.816167Z","shell.execute_reply":"2026-01-07T12:38:53.476299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.boxplot(x='Policy Type', y='Premium Amount', data=df)\nplt.title('Premium Amount by Policy Type')\nplt.show()           ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:53.478315Z","iopub.execute_input":"2026-01-07T12:38:53.479006Z","iopub.status.idle":"2026-01-07T12:38:55.380932Z","shell.execute_reply.started":"2026-01-07T12:38:53.478977Z","shell.execute_reply":"2026-01-07T12:38:55.379854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr = df[num].corr() # Create a heatmap showing correlations between all numerical variables.\nplt.figure(figsize=(12, 10))\nsns.heatmap(corr, annot=True, cmap='coolwarm', fmt='.2f')\nplt.title('Correlation Matrix')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:55.38199Z","iopub.execute_input":"2026-01-07T12:38:55.382592Z","iopub.status.idle":"2026-01-07T12:38:56.33143Z","shell.execute_reply.started":"2026-01-07T12:38:55.382563Z","shell.execute_reply":"2026-01-07T12:38:56.330587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(4, 3, figsize=(18, 20))\naxes = axes.flatten()\nfor i, col in enumerate(cat):\n    sns.countplot(y=col, data=df, ax=axes[i])\n    axes[i].set_title(f'{col}')\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:38:56.335057Z","iopub.execute_input":"2026-01-07T12:38:56.335335Z","iopub.status.idle":"2026-01-07T12:39:11.956918Z","shell.execute_reply.started":"2026-01-07T12:38:56.335312Z","shell.execute_reply":"2026-01-07T12:39:11.955906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['Year'] = df['Policy Start Date'].dt.year # Extract date components (Year, Month, Day, DayOfWeek) from the Policy Start Date column.\ndf['Month'] = df['Policy Start Date'].dt.month\ndf['Day'] = df['Policy Start Date'].dt.day\ndf['DayOfWeek'] = df['Policy Start Date'].dt.dayofweek","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:39:11.958135Z","iopub.execute_input":"2026-01-07T12:39:11.958617Z","iopub.status.idle":"2026-01-07T12:39:12.211454Z","shell.execute_reply.started":"2026-01-07T12:39:11.95858Z","shell.execute_reply":"2026-01-07T12:39:12.210396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(14, 10))\nsns.countplot(x='Year', data=df, ax=axes[0, 0])\nsns.countplot(x='Month', data=df, ax=axes[0, 1])\nsns.countplot(x='Day', data=df, ax=axes[1, 0])\nsns.countplot(x='DayOfWeek', data=df, ax=axes[1, 1])\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:39:12.212729Z","iopub.execute_input":"2026-01-07T12:39:12.213081Z","iopub.status.idle":"2026-01-07T12:39:20.374983Z","shell.execute_reply.started":"2026-01-07T12:39:12.213047Z","shell.execute_reply":"2026-01-07T12:39:20.37387Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"TEP 6: OUTLIER DETECTION","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 3, figsize=(15, 12))\naxes = axes.flatten()\nfor i, col in enumerate(num[:9]):\n    sns.boxplot(x=df[col], ax=axes[i])\n    axes[i].set_title(f'{col}')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:39:20.376202Z","iopub.execute_input":"2026-01-07T12:39:20.376666Z","iopub.status.idle":"2026-01-07T12:39:38.420241Z","shell.execute_reply.started":"2026-01-07T12:39:20.376637Z","shell.execute_reply":"2026-01-07T12:39:38.41925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in num: # Calculate and count outliers using the IQR (Interquartile Range) method.\n    Q1 = df[col].quantile(0.25)\n    Q3 = df[col].quantile(0.75)\n    IQR = Q3 - Q1\n    lower = Q1 - 1.5 * IQR\n    upper = Q3 + 1.5 * IQR\n    outliers = df[(df[col] < lower) | (df[col] > upper)]\n    print(f'{col}: {len(outliers)} outliers')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T12:39:38.42172Z","iopub.execute_input":"2026-01-07T12:39:38.422192Z","iopub.status.idle":"2026-01-07T12:39:38.89962Z","shell.execute_reply.started":"2026-01-07T12:39:38.422151Z","shell.execute_reply":"2026-01-07T12:39:38.89834Z"}},"outputs":[],"execution_count":null}]}