{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=Warning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:45.635136Z","iopub.execute_input":"2025-01-01T08:20:45.635475Z","iopub.status.idle":"2025-01-01T08:20:46.459665Z","shell.execute_reply.started":"2025-01-01T08:20:45.635444Z","shell.execute_reply":"2025-01-01T08:20:46.45895Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Loading Data","metadata":{}},{"cell_type":"code","source":"# sample_submission\nsample_submission = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\nprint(sample_submission.shape)\nsample_submission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:46.461397Z","iopub.execute_input":"2025-01-01T08:20:46.461895Z","iopub.status.idle":"2025-01-01T08:20:46.649285Z","shell.execute_reply.started":"2025-01-01T08:20:46.461853Z","shell.execute_reply":"2025-01-01T08:20:46.648362Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\nprint(test.shape)\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:46.650208Z","iopub.execute_input":"2025-01-01T08:20:46.650457Z","iopub.status.idle":"2025-01-01T08:20:49.01677Z","shell.execute_reply.started":"2025-01-01T08:20:46.650432Z","shell.execute_reply":"2025-01-01T08:20:49.015894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train\ntrain = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\nprint(train.shape)\ntrain.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:49.018418Z","iopub.execute_input":"2025-01-01T08:20:49.018718Z","iopub.status.idle":"2025-01-01T08:20:52.605455Z","shell.execute_reply.started":"2025-01-01T08:20:49.01869Z","shell.execute_reply":"2025-01-01T08:20:52.604504Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. train column description\n- 'id' 0 ~ 1199999\n- 'Age' 18 ~ 64\n- 'Gender' Male & Female\n- 'Annual Income' 1 ~ 149,997\n- 'Marital Status' Single, Married, Divorced\n- 'Number of Dependents' 0, 1, 2, 3, 4\n- 'Education Level' High School, Bachelor's, Master's, PhD\n- 'Occupation' Employed, Self-Employed, Unemployed   \n- 'Health Score' 2.012237 ~ 58.975914\n- 'Location' Rural, Suburban, Urban\n- 'Policy Type' Premium, Comprehensive, Basic \n- 'Previous Claims' 0.0 ~ 9.0\n- 'Vehicle Age' 0 ~ 9\n- 'Credit Score' 300 ~ 849\n- 'Insurance Duration' 1 ~ 9\n- 'Policy Start Date', -> year, month, day, date\n- 'Customer Feedback' Average, Poor, Good\n- 'Smoking Status' Yes, No\n- 'Exercise Frequency' Daily , Weekly. Monthly, Rarely\n- 'Property Type' House, Apartment, Condo\n- 'Premium Amount' Target 20 ~ 4,999","metadata":{}},{"cell_type":"markdown","source":"# 3. Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# information\nprint(train.shape)\ntrain.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:52.606905Z","iopub.execute_input":"2025-01-01T08:20:52.60761Z","iopub.status.idle":"2025-01-01T08:20:53.162341Z","shell.execute_reply.started":"2025-01-01T08:20:52.607561Z","shell.execute_reply":"2025-01-01T08:20:53.161528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# missing data ratio\ntrain_missing_ratio = train.isna().sum() / train.shape[0]\ntest_missing_ratio = test.isna().sum() / test.shape[0]\n\nmissing_data = pd.DataFrame({\n    'Missing_Ratio_Train': train_missing_ratio,\n    'Missing_Ratio_Test': test_missing_ratio\n})\n\nmissing_data.round(6)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:53.163295Z","iopub.execute_input":"2025-01-01T08:20:53.163535Z","iopub.status.idle":"2025-01-01T08:20:54.066503Z","shell.execute_reply.started":"2025-01-01T08:20:53.163511Z","shell.execute_reply":"2025-01-01T08:20:54.065528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Policy Start Date'][:5]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:54.067462Z","iopub.execute_input":"2025-01-01T08:20:54.067722Z","iopub.status.idle":"2025-01-01T08:20:54.073748Z","shell.execute_reply.started":"2025-01-01T08:20:54.067697Z","shell.execute_reply":"2025-01-01T08:20:54.072917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert 'Policy Start Date' string to datetime object and split\ntrain['year'] = pd.to_datetime(train['Policy Start Date']).dt.year\ntrain['month'] = pd.to_datetime(train['Policy Start Date']).dt.month\ntrain['day'] = pd.to_datetime(train['Policy Start Date']).dt.day\ntrain['day_of_week'] = pd.to_datetime(train['Policy Start Date']).dt.day_of_week\n\ntest['year'] = pd.to_datetime(test['Policy Start Date']).dt.year\ntest['month'] = pd.to_datetime(test['Policy Start Date']).dt.month\ntest['day'] = pd.to_datetime(test['Policy Start Date']).dt.day\ntest['day_of_week'] = pd.to_datetime(test['Policy Start Date']).dt.day_of_week","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:54.0751Z","iopub.execute_input":"2025-01-01T08:20:54.075856Z","iopub.status.idle":"2025-01-01T08:20:56.611194Z","shell.execute_reply.started":"2025-01-01T08:20:54.075825Z","shell.execute_reply":"2025-01-01T08:20:56.61021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Average by Condition function\ndef average_by_condition(df, numeric, condition):\n    return df.groupby(condition)[numeric].mean().sort_values(ascending=True)\n\n# Scatter plot function\ndef plot_scatter(df, x_col, y_col):\n    plt.figure(figsize=(3, 2))\n    plt.scatter(df[x_col], df[y_col], color='blue')\n    plt.xlabel(x_col)\n    plt.ylabel(y_col)\n    plt.grid(True)\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:56.612455Z","iopub.execute_input":"2025-01-01T08:20:56.612777Z","iopub.status.idle":"2025-01-01T08:20:56.618754Z","shell.execute_reply.started":"2025-01-01T08:20:56.612747Z","shell.execute_reply":"2025-01-01T08:20:56.617776Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-1. Age","metadata":{}},{"cell_type":"code","source":"# 1. scatter plot of Premium Amount Vs Age\nresult = average_by_condition(train, 'Premium Amount', 'Age')\ndf = pd.DataFrame({'Age': result.index, 'Ave Premium Amount': result.values})\n\nplot_scatter(df, 'Age', 'Ave Premium Amount')\n# The higher Age shows the lower Premium Amount.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:56.62247Z","iopub.execute_input":"2025-01-01T08:20:56.623358Z","iopub.status.idle":"2025-01-01T08:20:56.848982Z","shell.execute_reply.started":"2025-01-01T08:20:56.623313Z","shell.execute_reply":"2025-01-01T08:20:56.848215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Age histogram \nfrom scipy.stats import skew\nfig = plt.figure(figsize=(4, 2))\nplt.title(f'count Histogram\\nSkewness: {float(skew(train.Age)):.2f}')\nsns.histplot(train['Age'], kde=True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:20:56.849989Z","iopub.execute_input":"2025-01-01T08:20:56.850343Z","iopub.status.idle":"2025-01-01T08:21:01.631887Z","shell.execute_reply.started":"2025-01-01T08:20:56.850304Z","shell.execute_reply":"2025-01-01T08:21:01.630996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fill NaN\ntrain['Age'].fillna(train['Age'].mean(), inplace=True)\ntest['Age'].fillna(test['Age'].mean(), inplace=True)\ntrain['Age'].isna().sum(), test['Age'].isna().sum()   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:01.632874Z","iopub.execute_input":"2025-01-01T08:21:01.633122Z","iopub.status.idle":"2025-01-01T08:21:01.656497Z","shell.execute_reply.started":"2025-01-01T08:21:01.633096Z","shell.execute_reply":"2025-01-01T08:21:01.655643Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-2. Annual Income ","metadata":{}},{"cell_type":"code","source":"# 2. scatter plot of Premium Amount Vs Annual Income\nresult = average_by_condition(train, 'Premium Amount', 'Annual Income')\ndf = pd.DataFrame({'Annual Income': result.index, 'Ave Premium Amount': result.values})\n\nGroup = np.linspace(df['Annual Income'].min(), df['Annual Income'].max(), 11)\nlabels = [f'Group_{i}' for i in range(1, 11)]\ndf['Annual Income_G'] = pd.cut(df['Annual Income'], bins=Group, labels=labels, include_lowest=True)\ndf = df.groupby('Annual Income_G')[['Annual Income', 'Ave Premium Amount']].mean().reset_index()\n\nplt.figure(figsize=(4, 2))\nplt.scatter(df['Annual Income'], df['Ave Premium Amount'], color='blue')\nplt.xlabel('Annual Income')\nplt.ylabel('Ave Premium Amount')\nplt.grid(True)\nplt.show()\n# Premium Amount are cheaper up to annual income of 50,000, but above that, \n# as Annual Income increases, Premium Amount also increase.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:01.657462Z","iopub.execute_input":"2025-01-01T08:21:01.657753Z","iopub.status.idle":"2025-01-01T08:21:01.842636Z","shell.execute_reply.started":"2025-01-01T08:21:01.657726Z","shell.execute_reply":"2025-01-01T08:21:01.841827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Annual Income histogram \nfrom scipy.stats import skew\nfig = plt.figure(figsize=(6, 2))\nplt.title(f'count Histogram\\nSkewness: {float(skew(train[\"Annual Income\"])):.2f}')\nsns.histplot(train['Annual Income'], kde=True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:01.843628Z","iopub.execute_input":"2025-01-01T08:21:01.843973Z","iopub.status.idle":"2025-01-01T08:21:06.984119Z","shell.execute_reply.started":"2025-01-01T08:21:01.843935Z","shell.execute_reply":"2025-01-01T08:21:06.983231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill NaNs with the median to give the lowest skewness\nfrom sklearn.preprocessing import PowerTransformer\n\ntrain['Annual Income'] = train['Annual Income'].fillna(train['Annual Income'].median())\ntest['Annual Income'] = test['Annual Income'].fillna(test['Annual Income'].median())\n\n# Yeo-Johnson transformation\nincome_2d = train['Annual Income'].values.reshape(-1, 1)\ntest_income_2d = test['Annual Income'].values.reshape(-1, 1)\n\npt = PowerTransformer(method='yeo-johnson')\ntrain['Annual Income'] = pt.fit_transform(income_2d)\ntest['Annual Income'] = pt.transform(test_income_2d)\n\nfig = plt.figure(figsize=(6, 2))\nplt.title(f'count Histogram\\nSkewness: {float(skew(train[\"Annual Income\"])):.2f}')\nsns.histplot(train['Annual Income'], kde=True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:06.985176Z","iopub.execute_input":"2025-01-01T08:21:06.98541Z","iopub.status.idle":"2025-01-01T08:21:12.881226Z","shell.execute_reply.started":"2025-01-01T08:21:06.985387Z","shell.execute_reply":"2025-01-01T08:21:12.880412Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-3. Crtedit Score","metadata":{}},{"cell_type":"code","source":"# 3. scatter plot of Premium Amount Vs Crtedit Score\nresult = average_by_condition(train, 'Premium Amount', 'Credit Score')\ndf = pd.DataFrame({'Credit Score': result.index, 'Ave Premium Amount': result.values})\n\nGroup = np.linspace(df['Credit Score'].min(), df['Credit Score'].max(), 11)\nlabels = [f'Group_{i}' for i in range(1, 11)]\ndf['Credit Score_G'] = pd.cut(df['Credit Score'], bins=Group, labels=labels, include_lowest=True)\ndf = df.groupby('Credit Score_G')[['Credit Score', 'Ave Premium Amount']].mean().reset_index()\n\nplt.figure(figsize=(4, 2))\nplt.scatter(df['Credit Score'], df['Ave Premium Amount'], color='blue')\nplt.xlabel('Credit Score')\nplt.ylabel('Ave Premium Amount')\nplt.grid(True)\nplt.show()\n# the higher Credit Score shows the lower Premium Amount","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:12.8826Z","iopub.execute_input":"2025-01-01T08:21:12.883368Z","iopub.status.idle":"2025-01-01T08:21:13.042348Z","shell.execute_reply.started":"2025-01-01T08:21:12.883325Z","shell.execute_reply":"2025-01-01T08:21:13.04148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill NaN\ntrain['Credit Score'] = train['Credit Score'].fillna(train['Credit Score'].mean())\ntest['Credit Score'] = test['Credit Score'].fillna(test['Credit Score'].mean())\n\n# Credit Score histogram \nfig = plt.figure(figsize=(4, 2))\nplt.title(f'count Histogram\\nSkewness: {float(skew(train[\"Credit Score\"])):.2f}')\nsns.histplot(train['Credit Score'], kde=True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:13.043516Z","iopub.execute_input":"2025-01-01T08:21:13.043901Z","iopub.status.idle":"2025-01-01T08:21:17.732189Z","shell.execute_reply.started":"2025-01-01T08:21:13.043857Z","shell.execute_reply":"2025-01-01T08:21:17.731287Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-4. Customer Feedback","metadata":{}},{"cell_type":"code","source":"# 4. Premium Amount Customer Feedback\naverage_by_condition(train, 'Premium Amount', 'Customer Feedback')\n# Premium Amount: Average > Good > Poor, Averge shows the lowest ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:17.733365Z","iopub.execute_input":"2025-01-01T08:21:17.733746Z","iopub.status.idle":"2025-01-01T08:21:17.83419Z","shell.execute_reply.started":"2025-01-01T08:21:17.733716Z","shell.execute_reply":"2025-01-01T08:21:17.833364Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill NaN as Unknown\ntrain['Customer Feedback'] = train['Customer Feedback'].fillna('Unknown')\ntest['Customer Feedback'] = test['Customer Feedback'].fillna('Unknown')\ntrain['Customer Feedback'].isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:17.835564Z","iopub.execute_input":"2025-01-01T08:21:17.835915Z","iopub.status.idle":"2025-01-01T08:21:17.997423Z","shell.execute_reply.started":"2025-01-01T08:21:17.835875Z","shell.execute_reply":"2025-01-01T08:21:17.99654Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-5. Education Level","metadata":{}},{"cell_type":"code","source":"# 5. Premium Amount Vs Education Level\naverage_by_condition(train, 'Premium Amount', 'Education Level')\n# Premium Amount: High School > Bachelor's > Master's > PhD ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:17.998344Z","iopub.execute_input":"2025-01-01T08:21:17.998593Z","iopub.status.idle":"2025-01-01T08:21:18.085272Z","shell.execute_reply.started":"2025-01-01T08:21:17.998569Z","shell.execute_reply":"2025-01-01T08:21:18.084334Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-6. Exercise Frequency","metadata":{}},{"cell_type":"code","source":"# 6. Premium Amount Vs Exercise Frequency\naverage_by_condition(train, 'Premium Amount', 'Exercise Frequency')\n# Premium Amount: Daily > Monthly > Rarely > Weekly","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:18.086282Z","iopub.execute_input":"2025-01-01T08:21:18.086588Z","iopub.status.idle":"2025-01-01T08:21:18.164143Z","shell.execute_reply.started":"2025-01-01T08:21:18.086558Z","shell.execute_reply":"2025-01-01T08:21:18.163268Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-7. Gender ","metadata":{}},{"cell_type":"code","source":"# 7. Premium Amount Vs Gender \naverage_by_condition(train, 'Premium Amount', 'Gender')\n# Premium Amount: Male > Female","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:18.165102Z","iopub.execute_input":"2025-01-01T08:21:18.165405Z","iopub.status.idle":"2025-01-01T08:21:18.240794Z","shell.execute_reply.started":"2025-01-01T08:21:18.165377Z","shell.execute_reply":"2025-01-01T08:21:18.239832Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-8. Health Score","metadata":{}},{"cell_type":"code","source":"# 8. Premium Amount Vs Health Score\nresult = average_by_condition(train, 'Premium Amount', 'Health Score')\ndf = pd.DataFrame({'Health Score': result.index, 'Premium Amount': result.values})\n\nGroup = np.linspace(df['Health Score'].min(), df['Health Score'].max(), 11)\nlabels = [f'Group_{i}' for i in range(1, 11)]\ndf['Health_Score_G'] = pd.cut(df['Health Score'], bins=Group, labels=labels, include_lowest=True)\ndf = df.groupby('Health_Score_G')[['Health Score', 'Premium Amount']].mean().reset_index()\n\nplt.figure(figsize=(4, 2))\nplt.scatter(df['Health Score'], df['Premium Amount'], color='blue')\nplt.xlabel('Health Score')\nplt.ylabel('Ave Premium Amount')\nplt.grid(True)\nplt.show()\n# Premium Amount are cheaper up to Health Score of 30, but above that, \n# as Health Score increases, Premium Amount also increase.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:18.241852Z","iopub.execute_input":"2025-01-01T08:21:18.242115Z","iopub.status.idle":"2025-01-01T08:21:18.750916Z","shell.execute_reply.started":"2025-01-01T08:21:18.242089Z","shell.execute_reply":"2025-01-01T08:21:18.750076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill NaN\ntrain['Health Score'] = train['Health Score'].fillna(train['Health Score'].mean())\ntest['Health Score'] = test['Health Score'].fillna(test['Health Score'].mean())\n\n# Credit Score histogram \nfig = plt.figure(figsize=(4, 2))\nplt.title(f'count Histogram\\nSkewness: {float(skew(train[\"Health Score\"])):.2f}')\nsns.histplot(train['Health Score'], kde=True)\nplt.show()\n# 0.29","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:18.752121Z","iopub.execute_input":"2025-01-01T08:21:18.7525Z","iopub.status.idle":"2025-01-01T08:21:23.762621Z","shell.execute_reply.started":"2025-01-01T08:21:18.752442Z","shell.execute_reply":"2025-01-01T08:21:23.761798Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-9. Insurance Duration","metadata":{}},{"cell_type":"code","source":"# 9. scatter plot of Premium Amount Vs Insurance Duration\nresult = average_by_condition(train, 'Premium Amount', 'Insurance Duration')\ndf = pd.DataFrame({'Insurance Duration': result.index, 'Ave Premium Amount': result.values})\n\nplot_scatter(df, 'Insurance Duration', 'Ave Premium Amount')\n# The longer Insurance Duration shows the higher Premium Amount.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:23.763942Z","iopub.execute_input":"2025-01-01T08:21:23.764309Z","iopub.status.idle":"2025-01-01T08:21:23.972625Z","shell.execute_reply.started":"2025-01-01T08:21:23.764248Z","shell.execute_reply":"2025-01-01T08:21:23.971741Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill NaN with mean\ntrain['Insurance Duration'] = train['Insurance Duration'].fillna(train['Insurance Duration'].mean())\ntest['Insurance Duration'] = test['Insurance Duration'].fillna(test['Insurance Duration'].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:23.973638Z","iopub.execute_input":"2025-01-01T08:21:23.97391Z","iopub.status.idle":"2025-01-01T08:21:23.995571Z","shell.execute_reply.started":"2025-01-01T08:21:23.973877Z","shell.execute_reply":"2025-01-01T08:21:23.994853Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-10. Location","metadata":{}},{"cell_type":"code","source":"# 10. Premium Amount Vs Location\naverage_by_condition(train, 'Premium Amount', 'Location')\n# Premium Amount: Urban > Suburban >  Rural , possibility of accident","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:23.996638Z","iopub.execute_input":"2025-01-01T08:21:23.996912Z","iopub.status.idle":"2025-01-01T08:21:24.070365Z","shell.execute_reply.started":"2025-01-01T08:21:23.996885Z","shell.execute_reply":"2025-01-01T08:21:24.069534Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-11. Marital Status","metadata":{}},{"cell_type":"code","source":"# 11. Premium Amount Vs Marital Status\naverage_by_condition(train, 'Premium Amount', 'Marital Status')\n# Premium Amount: Married > Divorced >  Single","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:24.076495Z","iopub.execute_input":"2025-01-01T08:21:24.076841Z","iopub.status.idle":"2025-01-01T08:21:24.180881Z","shell.execute_reply.started":"2025-01-01T08:21:24.076811Z","shell.execute_reply":"2025-01-01T08:21:24.1799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill NaN as Unknown\ntrain['Marital Status'] = train['Marital Status'].fillna('Unknown')\ntest['Marital Status'] = test['Marital Status'].fillna('Unknown')\ntrain['Customer Feedback'].isna().sum()\n# Premium Amount: Married > Unknown > Divorced >  Single","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:24.182177Z","iopub.execute_input":"2025-01-01T08:21:24.182618Z","iopub.status.idle":"2025-01-01T08:21:24.346438Z","shell.execute_reply.started":"2025-01-01T08:21:24.182572Z","shell.execute_reply":"2025-01-01T08:21:24.345544Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-12. Number of Dependents","metadata":{}},{"cell_type":"code","source":"# 12. Premium Amount Vs Number of Dependents\naverage_by_condition(train, 'Premium Amount', 'Number of Dependents')\n# discount above 3 Dependents","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:24.347429Z","iopub.execute_input":"2025-01-01T08:21:24.347731Z","iopub.status.idle":"2025-01-01T08:21:24.380706Z","shell.execute_reply.started":"2025-01-01T08:21:24.347704Z","shell.execute_reply":"2025-01-01T08:21:24.379796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fill NaN\ntrain['Number of Dependents'].fillna(3, inplace=True)\ntest['Number of Dependents'].fillna(3, inplace=True)\ntrain['Number of Dependents'].isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:24.381737Z","iopub.execute_input":"2025-01-01T08:21:24.382025Z","iopub.status.idle":"2025-01-01T08:21:24.396779Z","shell.execute_reply.started":"2025-01-01T08:21:24.381997Z","shell.execute_reply":"2025-01-01T08:21:24.396045Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-13. Occupation ","metadata":{}},{"cell_type":"code","source":"# 13. Premium Amount Vs Occupation\naverage_by_condition(train, 'Premium Amount', 'Occupation')\n# Premium Amount: Employed > Self-Employed >  Unemployed","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:24.397728Z","iopub.execute_input":"2025-01-01T08:21:24.397965Z","iopub.status.idle":"2025-01-01T08:21:24.494029Z","shell.execute_reply.started":"2025-01-01T08:21:24.397941Z","shell.execute_reply":"2025-01-01T08:21:24.493198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill NaN as Unknown\ntrain['Occupation'] = train['Occupation'].fillna('Unknown')\ntest['Occupation'] = test['Occupation'].fillna('Unknown')\ntrain['Occupation'].isna().sum(), test['Occupation'].isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:24.494945Z","iopub.execute_input":"2025-01-01T08:21:24.495204Z","iopub.status.idle":"2025-01-01T08:21:24.695519Z","shell.execute_reply.started":"2025-01-01T08:21:24.495172Z","shell.execute_reply":"2025-01-01T08:21:24.694533Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-14. Policy Start Date - (year, month, day, day_of_week)","metadata":{}},{"cell_type":"code","source":"# 14. Premium Amount Vs Policy Start Date - (year, month, day, day_of_week)\naverage_by_condition(train, 'Premium Amount', 'day_of_week') # year, month, day, day_of_week","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:24.696457Z","iopub.execute_input":"2025-01-01T08:21:24.696723Z","iopub.status.idle":"2025-01-01T08:21:24.721946Z","shell.execute_reply.started":"2025-01-01T08:21:24.696697Z","shell.execute_reply":"2025-01-01T08:21:24.721066Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-15. Policy Type","metadata":{}},{"cell_type":"code","source":"# 15. Premium Amount Vs Policy Type\naverage_by_condition(train, 'Premium Amount', 'Policy Type')\n# Premium Amount: Basic > Comprehensive >  Premium , Group's risk difference ?","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:24.723032Z","iopub.execute_input":"2025-01-01T08:21:24.723275Z","iopub.status.idle":"2025-01-01T08:21:24.803175Z","shell.execute_reply.started":"2025-01-01T08:21:24.723251Z","shell.execute_reply":"2025-01-01T08:21:24.802281Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-16. Previous Claims","metadata":{}},{"cell_type":"code","source":"# 16. Previous Claims\nresult = average_by_condition(train, 'Premium Amount', 'Previous Claims')\ndf = pd.DataFrame({'Previous Claims': result.index, 'Ave Premium Amount': result.values})\n\nplot_scatter(df, 'Previous Claims', 'Ave Premium Amount')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:24.804191Z","iopub.execute_input":"2025-01-01T08:21:24.80447Z","iopub.status.idle":"2025-01-01T08:21:25.02132Z","shell.execute_reply.started":"2025-01-01T08:21:24.804441Z","shell.execute_reply":"2025-01-01T08:21:25.020385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill NaN\ntrain['Previous Claims'] = train['Previous Claims'].fillna(train['Previous Claims'].mean())\ntest['Previous Claims'] = test['Previous Claims'].fillna(test['Previous Claims'].mean())\n\n# Credit Score histogram \nfig = plt.figure(figsize=(4, 2))\nplt.title(f'count Histogram\\nSkewness: {float(skew(train[\"Previous Claims\"])):.2f}')\nsns.histplot(train['Previous Claims'], kde=True)\nplt.show()\n# 1.08","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:25.02234Z","iopub.execute_input":"2025-01-01T08:21:25.022611Z","iopub.status.idle":"2025-01-01T08:21:30.396354Z","shell.execute_reply.started":"2025-01-01T08:21:25.022586Z","shell.execute_reply":"2025-01-01T08:21:30.395527Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-17. Property Type","metadata":{}},{"cell_type":"code","source":"# 17. Premium Amount Vs Property Type\naverage_by_condition(train, 'Premium Amount', 'Property Type')\n# Premium Amount: Apartment > Condo >  House","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:30.397686Z","iopub.execute_input":"2025-01-01T08:21:30.398078Z","iopub.status.idle":"2025-01-01T08:21:30.472438Z","shell.execute_reply.started":"2025-01-01T08:21:30.398035Z","shell.execute_reply":"2025-01-01T08:21:30.471669Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-18. Smoking Status","metadata":{}},{"cell_type":"code","source":"# 18. Premium Amount Vs Smoking Status\naverage_by_condition(train, 'Premium Amount', 'Smoking Status')\n# smokers pay more","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:30.473403Z","iopub.execute_input":"2025-01-01T08:21:30.47398Z","iopub.status.idle":"2025-01-01T08:21:30.545775Z","shell.execute_reply.started":"2025-01-01T08:21:30.473948Z","shell.execute_reply":"2025-01-01T08:21:30.54492Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3-19. Vehicle Age","metadata":{}},{"cell_type":"code","source":"# 19. Scatter plot of Premium Amount Vs Vehicle Age\nresult = average_by_condition(train, 'Premium Amount', 'Vehicle Age')\ndf = pd.DataFrame({'Vehicle Age': result.index, 'Ave Premium Amount': result.values})\nplot_scatter(df, 'Vehicle Age', 'Ave Premium Amount')\n# The higher Vehicle Age shows the higher Premium Amount.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:30.546941Z","iopub.execute_input":"2025-01-01T08:21:30.547341Z","iopub.status.idle":"2025-01-01T08:21:30.76644Z","shell.execute_reply.started":"2025-01-01T08:21:30.547289Z","shell.execute_reply":"2025-01-01T08:21:30.765538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill NaN\ntrain['Vehicle Age'] = train['Vehicle Age'].fillna(train['Vehicle Age'].mean())\ntest['Vehicle Age'] = test['Vehicle Age'].fillna(test['Vehicle Age'].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:30.767589Z","iopub.execute_input":"2025-01-01T08:21:30.767867Z","iopub.status.idle":"2025-01-01T08:21:30.790756Z","shell.execute_reply.started":"2025-01-01T08:21:30.767841Z","shell.execute_reply":"2025-01-01T08:21:30.79Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# features, target & test_df\nfeatures = train.drop([\"id\", \"Premium Amount\", \"Policy Start Date\"], axis=1)\ntarget = train[[\"Premium Amount\"]]\ntest_df = test.drop([\"id\", \"Policy Start Date\"], axis=1)\n\nfeatures.shape, target.shape, test_df.shape\n# (1200000, 22), (1200000, 1), (800000, 22)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:30.79202Z","iopub.execute_input":"2025-01-01T08:21:30.792289Z","iopub.status.idle":"2025-01-01T08:21:31.104282Z","shell.execute_reply.started":"2025-01-01T08:21:30.792263Z","shell.execute_reply":"2025-01-01T08:21:31.103397Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. EDA","metadata":{}},{"cell_type":"markdown","source":"## 4-1. target data","metadata":{}},{"cell_type":"code","source":"# target histogram \nfrom scipy.stats import skew\n\n# count_target histogram \nfig = plt.figure(figsize=(4, 2))\n\nplt.title(f'count Histogram\\nSkewness: {float(skew(target)):.2f}')\nsns.histplot(target, kde=True)\n\nplt.show()\n# skewness is less than 2.0 so let's use as is.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:31.105382Z","iopub.execute_input":"2025-01-01T08:21:31.10567Z","iopub.status.idle":"2025-01-01T08:21:37.717802Z","shell.execute_reply.started":"2025-01-01T08:21:31.105644Z","shell.execute_reply":"2025-01-01T08:21:37.716902Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4-2. numerical data","metadata":{}},{"cell_type":"code","source":"# numeric variables in features df\nfeatures_num = features.select_dtypes(include=['int', 'float']).columns\nfeatures_num = features[features_num]\n\ntest_num = test_df.select_dtypes(include=['int', 'float']).columns\ntest_num = test_df[test_num]\n\nfeatures_num.shape, test_num.shape\n# (1200000, 12), (800000, 12)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:37.718805Z","iopub.execute_input":"2025-01-01T08:21:37.719053Z","iopub.status.idle":"2025-01-01T08:21:37.978822Z","shell.execute_reply.started":"2025-01-01T08:21:37.719029Z","shell.execute_reply":"2025-01-01T08:21:37.977906Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4-3. generate numerical features","metadata":{}},{"cell_type":"code","source":" features_num.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:37.980367Z","iopub.execute_input":"2025-01-01T08:21:37.980749Z","iopub.status.idle":"2025-01-01T08:21:37.986286Z","shell.execute_reply.started":"2025-01-01T08:21:37.980709Z","shell.execute_reply":"2025-01-01T08:21:37.985435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# features_num\nfeatures_num['Income_to_Age'] = features_num['Annual Income'] / features_num['Age']\n\nfeatures_num['Health_Score_to_Age'] = features_num['Health Score'] / features_num['Age']\n\nfeatures_num['Is_Retired'] = (features_num['Age'] >= 60).astype(int)\n\nfeatures_num['Income_per_Family_Member'] = (features_num['Annual Income'] / \n                                            (features_num['Number of Dependents'] + 1)) \n\nfeatures_num['Claim Frequency'] = features_num['Previous Claims'] / features_num['Insurance Duration']\n\nfeatures_num['Risk_Score'] = (features_num['Age'] + features_num['Health Score'] + \n                              features_num['Credit Score'] + features_num['Previous Claims'])\n# test_num\ntest_num['Income_to_Age'] = test_num['Annual Income'] / test_num['Age']\n\ntest_num['Health_Score_to_Age'] = test_num['Health Score'] / test_num['Age']\n\ntest_num['Is_Retired'] = (test_num['Age'] >= 60).astype(int)\n\ntest_num['Income_per_Family_Member'] = (test_num['Annual Income'] / \n                                        (test_num['Number of Dependents'] + 1)) \n\ntest_num['Claim Frequency'] = test_num['Previous Claims'] / test_num['Insurance Duration']\n\ntest_num['Risk_Score'] = (test_num['Age'] + test_num['Health Score'] + \n                          test_num['Credit Score'] + test_num['Previous Claims'])\n\nfeatures_num.shape, test_num.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:37.987274Z","iopub.execute_input":"2025-01-01T08:21:37.987563Z","iopub.status.idle":"2025-01-01T08:21:38.077278Z","shell.execute_reply.started":"2025-01-01T08:21:37.987536Z","shell.execute_reply":"2025-01-01T08:21:38.076456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# features_num_squared \nfeatures_num_columns = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score', \n                        'Previous Claims', 'Vehicle Age', 'Credit Score', 'Insurance Duration', \n                        'year', 'month', 'day', 'day_of_week']\n\ntest_num_columns = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score', \n                        'Previous Claims', 'Vehicle Age', 'Credit Score', 'Insurance Duration', \n                        'year', 'month', 'day', 'day_of_week']\n\n# features_num_squared \nfor col in features_num_columns:\n    features_num[f'{col}_squared'] = features_num[col] ** 2\n\n# test_num_squared \nfor col in test_num_columns:\n    test_num[f'{col}_squared'] = test_num[col] ** 2\n    \nfeatures_num.shape, test_num.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:38.078491Z","iopub.execute_input":"2025-01-01T08:21:38.078917Z","iopub.status.idle":"2025-01-01T08:21:38.368013Z","shell.execute_reply.started":"2025-01-01T08:21:38.078872Z","shell.execute_reply":"2025-01-01T08:21:38.367169Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4-4. Standardization ","metadata":{}},{"cell_type":"code","source":"# Standardization\nfrom sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()\nscaler.fit(features_num[features_num.columns])\n\nfeatures_num_st = scaler.transform(features_num[features_num.columns])\nfeatures_num_st = pd.DataFrame(features_num_st, columns = features_num.columns)\n\ntest_num_st = scaler.transform(test_num[test_num.columns])\ntest_num_st = pd.DataFrame(test_num_st, columns = test_num.columns)\n\nprint(features_num_st.shape, test_num_st.shape)\nfeatures_num_st.head().T\n# (1200000, 14) (800000, 14)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:38.368988Z","iopub.execute_input":"2025-01-01T08:21:38.369249Z","iopub.status.idle":"2025-01-01T08:21:39.666059Z","shell.execute_reply.started":"2025-01-01T08:21:38.369225Z","shell.execute_reply":"2025-01-01T08:21:39.66514Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4-5. categorical data","metadata":{}},{"cell_type":"code","source":"# categoric variables in features\nfeatures_cat = features.select_dtypes(include=['object']).columns\nfeatures_cat = features[features_cat]\n\ntest_cat = test_df.select_dtypes(include=['object']).columns\ntest_cat = test_df[test_cat]\n\nfeatures_cat.shape, test_cat.shape\n# (1200000, 10), (800000, 10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:39.667099Z","iopub.execute_input":"2025-01-01T08:21:39.667454Z","iopub.status.idle":"2025-01-01T08:21:40.589268Z","shell.execute_reply.started":"2025-01-01T08:21:39.667424Z","shell.execute_reply":"2025-01-01T08:21:40.588387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_cat","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:40.590273Z","iopub.execute_input":"2025-01-01T08:21:40.590543Z","iopub.status.idle":"2025-01-01T08:21:40.603251Z","shell.execute_reply.started":"2025-01-01T08:21:40.590501Z","shell.execute_reply":"2025-01-01T08:21:40.602285Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4-6. generate categorical features","metadata":{}},{"cell_type":"code","source":"# features_cat \nfeatures_cat['Health Score_Group'] = pd.cut(train['Health Score'], bins=[0, 50, 100], \n                                           labels=['Low_Health Score', 'High_Health Score'])\n\n# test_cat \ntest_cat['Health Score_Group'] = pd.cut(test['Health Score'], bins=[0, 50, 100], \n                                           labels=['Low_Health Score', 'High_Health Score'])\n\nfeatures_cat.shape, test_cat.shape\n# ((1200000, 13), (800000, 13))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:40.604536Z","iopub.execute_input":"2025-01-01T08:21:40.604904Z","iopub.status.idle":"2025-01-01T08:21:40.65039Z","shell.execute_reply.started":"2025-01-01T08:21:40.604862Z","shell.execute_reply":"2025-01-01T08:21:40.649372Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4-7. ohe","metadata":{}},{"cell_type":"code","source":"# Check if the categorical data in train and test have the same class.\nprint(\"Features Categories:\")\nfor column in features_cat.columns:\n    print(f\"{column} unique? {features_cat[column].nunique()}\")\n    features_unique = set(features_cat[column].unique())\n    \n    if column in test_cat.columns:\n        test_unique = set(test_cat[column].unique())\n        is_same = features_unique == test_unique\n        print(f\"Same unique values as test set? {is_same}\")\n        if not is_same:\n            print(f\"Features unique: {features_unique}\")\n            print(f\"Test unique: {test_unique}\")\n            print(f\"Features only: {features_unique - test_unique}\")\n            print(f\"Test only: {test_unique - features_unique}\")\n    else:\n        print(f\"Column {column} not found in test set\")\n    print()\n# it has the same class so let's do ohe.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:40.651592Z","iopub.execute_input":"2025-01-01T08:21:40.651893Z","iopub.status.idle":"2025-01-01T08:21:42.161562Z","shell.execute_reply.started":"2025-01-01T08:21:40.651866Z","shell.execute_reply":"2025-01-01T08:21:42.160627Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# one_hot_encoding \nfeatures_cat_ohe = pd.get_dummies(features_cat, columns = features_cat.columns, \n                                 dtype=int, drop_first=True)\ntest_cat_ohe = pd.get_dummies(test_cat, columns = test_cat.columns, \n                                 dtype=int, drop_first=True)\nprint(features_cat_ohe.shape, test_cat_ohe.shape)\nfeatures_cat_ohe.head(2)\n# (1200000, 29) (800000, 29)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:42.162864Z","iopub.execute_input":"2025-01-01T08:21:42.163253Z","iopub.status.idle":"2025-01-01T08:21:43.870822Z","shell.execute_reply.started":"2025-01-01T08:21:42.16321Z","shell.execute_reply":"2025-01-01T08:21:43.869908Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4-8. review transformed data","metadata":{}},{"cell_type":"code","source":"# tranformed data\nfeatures_tf = pd.concat([features_num_st, features_cat_ohe], axis=1)\ntest_tf = pd.concat([test_num_st, test_cat_ohe], axis=1)\nfeatures_tf.shape, test_tf.shape, target.shape\n# (1200000, 59), (800000, 59), (1200000, 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:43.871849Z","iopub.execute_input":"2025-01-01T08:21:43.872114Z","iopub.status.idle":"2025-01-01T08:21:44.417599Z","shell.execute_reply.started":"2025-01-01T08:21:43.872088Z","shell.execute_reply":"2025-01-01T08:21:44.416744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dataframe target and features_tf\ntrain_tf = pd.concat([target, features_tf], axis=1)\nprint(train_tf.shape)\ntrain_tf.head()\n# (1200000, 55)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:44.418589Z","iopub.execute_input":"2025-01-01T08:21:44.41885Z","iopub.status.idle":"2025-01-01T08:21:44.614474Z","shell.execute_reply.started":"2025-01-01T08:21:44.418824Z","shell.execute_reply":"2025-01-01T08:21:44.613661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check NaN\ntrain_tf.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:44.61578Z","iopub.execute_input":"2025-01-01T08:21:44.616135Z","iopub.status.idle":"2025-01-01T08:21:44.698633Z","shell.execute_reply.started":"2025-01-01T08:21:44.616096Z","shell.execute_reply":"2025-01-01T08:21:44.697726Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4-9. distribution","metadata":{}},{"cell_type":"code","source":"# distribution \nfig, axes = plt.subplots(nrows=10, ncols=6, figsize=(20, 18))\naxes = axes.flatten()  \n\nfor i, col in enumerate(train_tf.columns):\n    axes[i].hist(train_tf[col], bins=20, color='blue', alpha=0.7)\n    axes[i].set_title(col)\n\n    skewness_value = skew(train_tf[col])\n    axes[i].text(0.95, 0.95, f'Skew: {skewness_value:.2f}', \n                 transform=axes[i].transAxes, fontsize=10,\n                 verticalalignment='top', horizontalalignment='right', color='red')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:44.700063Z","iopub.execute_input":"2025-01-01T08:21:44.70036Z","iopub.status.idle":"2025-01-01T08:21:56.385251Z","shell.execute_reply.started":"2025-01-01T08:21:44.700332Z","shell.execute_reply":"2025-01-01T08:21:56.384305Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4-10. correlation","metadata":{}},{"cell_type":"code","source":"# correlation with target \ncorrmat = train_tf.corr()\nsorted_corrmat = corrmat.iloc[:, 0].sort_values(ascending=False)\nsorted_corrmat","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:21:56.386343Z","iopub.execute_input":"2025-01-01T08:21:56.386623Z","iopub.status.idle":"2025-01-01T08:22:05.11092Z","shell.execute_reply.started":"2025-01-01T08:21:56.386596Z","shell.execute_reply":"2025-01-01T08:22:05.109897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# heatmap plot \nplt.figure(figsize=(16, 16))\nk=55\ncols = corrmat.nlargest(k, 'Premium Amount')['Premium Amount'].index\ncm = np.corrcoef(train_tf[cols].values.T)\nsns.set(font_scale=0.8)\nhm = sns.heatmap(cm, cbar=False, annot=True, square=True, fmt='.2f', annot_kws={'size': 8}, \n                 yticklabels=cols.values, xticklabels=cols.values, cmap=\"Purples\")\nhm.xaxis.tick_top()\nplt.xticks(rotation=45, ha='left')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:22:05.112112Z","iopub.execute_input":"2025-01-01T08:22:05.112366Z","iopub.status.idle":"2025-01-01T08:22:11.793075Z","shell.execute_reply.started":"2025-01-01T08:22:05.11234Z","shell.execute_reply":"2025-01-01T08:22:11.792279Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Modeling","metadata":{}},{"cell_type":"code","source":"# Set up mixed precision training (Step 1)\nimport tensorflow as tf\n\ntf.keras.mixed_precision.set_global_policy('mixed_float16')\n# mixing float 32 and 16 to speed up","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:22:11.794315Z","iopub.execute_input":"2025-01-01T08:22:11.794941Z","iopub.status.idle":"2025-01-01T08:22:14.520813Z","shell.execute_reply.started":"2025-01-01T08:22:11.7949Z","shell.execute_reply":"2025-01-01T08:22:14.520043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the create_dataset function (Step 2)\ndef create_dataset(X, y, batch_size=128):\n    dataset = tf.data.Dataset.from_tensor_slices((X, y))\n    return dataset.shuffle(buffer_size=1000).batch(batch_size).prefetch(tf.data.AUTOTUNE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:22:14.521874Z","iopub.execute_input":"2025-01-01T08:22:14.522386Z","iopub.status.idle":"2025-01-01T08:22:14.526894Z","shell.execute_reply.started":"2025-01-01T08:22:14.522357Z","shell.execute_reply":"2025-01-01T08:22:14.526067Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the RMSLE metric (Step 3)\ndef rmsle(y_true, y_pred):\n    y_true = tf.cast(y_true, tf.float32)\n    y_pred = tf.cast(y_pred, tf.float32)\n    \n    y_true = tf.maximum(y_true, 0)\n    y_pred = tf.maximum(y_pred, 0)\n    \n    y_true = y_true + 1 # avoid NaN\n    y_pred = y_pred + 1\n    \n    log_diff = tf.math.log(y_pred) - tf.math.log(y_true)\n    return tf.sqrt(tf.reduce_mean(tf.square(log_diff)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:22:14.527918Z","iopub.execute_input":"2025-01-01T08:22:14.528205Z","iopub.status.idle":"2025-01-01T08:22:14.536588Z","shell.execute_reply.started":"2025-01-01T08:22:14.528177Z","shell.execute_reply":"2025-01-01T08:22:14.535884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the model creation function (Step 4)\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.regularizers import l1_l2\nfrom tensorflow.keras.optimizers import Adam\n    \ndef create_model(input_dim):\n    model = Sequential([\n        Dense(256, activation='relu', input_dim=input_dim, kernel_regularizer=l1_l2(l1=1e-5, l2=1e-4)),\n        BatchNormalization(),\n        Dropout(0.3),\n        Dense(128, activation='relu', kernel_regularizer=l1_l2(l1=1e-5, l2=1e-4)),\n        BatchNormalization(),\n        Dropout(0.3),\n        Dense(64, activation='relu', kernel_regularizer=l1_l2(l1=1e-5, l2=1e-4)),\n        BatchNormalization(),\n        Dropout(0.2),\n        Dense(32, activation='relu', kernel_regularizer=l1_l2(l1=1e-5, l2=1e-4)),\n        BatchNormalization(),\n        Dropout(0.2),\n        Dense(1)\n    ])\n    \n    optimizer = Adam(learning_rate=0.001)\n    model.compile(optimizer=optimizer, loss=rmsle, metrics=[rmsle])\n    return model\n    \n    optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n    model.compile(optimizer=optimizer, loss=rmsle, metrics=[rmsle])\n    return model\n\ncreate_model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:22:14.537453Z","iopub.execute_input":"2025-01-01T08:22:14.537727Z","iopub.status.idle":"2025-01-01T08:22:14.551953Z","shell.execute_reply.started":"2025-01-01T08:22:14.537701Z","shell.execute_reply":"2025-01-01T08:22:14.551143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Perform KFold cross-validation and model training (Step 5)\nfrom sklearn.model_selection import KFold\n\nn_folds = 3\nkf = KFold(n_splits=n_folds, shuffle=True, random_state=2412)\n\ncv_results = []  \n\nfor fold, (train_index, val_index) in enumerate(kf.split(features_tf)):\n    print(f\"Fold {fold + 1}\")\n    \n    X_train, X_val = features_tf.iloc[train_index], features_tf.iloc[val_index]\n    y_train, y_val = target.iloc[train_index], target.iloc[val_index]\n    \n    train_dataset = create_dataset(X_train, y_train)\n    val_dataset = create_dataset(X_val, y_val)\n    \n    model = create_model(X_train.shape[1])\n\n    callbacks = [\n        tf.keras.callbacks.ReduceLROnPlateau(monitor='val_rmsle', factor=0.1, patience=10, \n                                             min_lr=0.01, mode='min'),\n        tf.keras.callbacks.EarlyStopping(monitor='val_rmsle', patience=10, verbose=1, mode='min')\n    ]\n    \n    history = model.fit(\n        train_dataset,\n        epochs=30,\n        validation_data=val_dataset,\n        callbacks=callbacks,\n        verbose=0 \n    )\n    \n    cv_results.append(history)\n    \n    print(\"Epoch  Train RMSLE  Val RMSLE\")\n    for epoch, (train_rmsle, val_rmsle) in enumerate(zip(history.history['rmsle'], \n                                                         history.history['val_rmsle']), 1):\n        print(f\"{epoch:5d}  {train_rmsle:.6f}  {val_rmsle:.6f}\")\n    \n    print(f\"\\nFinal Train RMSLE: {history.history['rmsle'][-1]:.6f}\")\n    print(f\"Final Val RMSLE: {history.history['val_rmsle'][-1]:.6f}\")\n    print(\"\\n\" + \"=\"*40 + \"\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:22:14.552961Z","iopub.execute_input":"2025-01-01T08:22:14.553285Z","iopub.status.idle":"2025-01-01T08:42:07.041916Z","shell.execute_reply.started":"2025-01-01T08:22:14.553248Z","shell.execute_reply":"2025-01-01T08:42:07.040899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(history.history.keys())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:42:07.043017Z","iopub.execute_input":"2025-01-01T08:42:07.043295Z","iopub.status.idle":"2025-01-01T08:42:07.048428Z","shell.execute_reply.started":"2025-01-01T08:42:07.043268Z","shell.execute_reply":"2025-01-01T08:42:07.047417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Summarize the results (Step 6)\n\nrmsles = [h.history['rmsle'][-1] for h in cv_results]\nval_rmsles = [h.history['val_rmsle'][-1] for h in cv_results]\n\nprint(f\"Average train RMSLE: {np.mean(rmsles):.4f}\")\nprint(f\"Average Val RMSLE: {np.mean(val_rmsles):.4f}\")\n\n# Average train RMSLE: 1.0617\n# Average Val RMSLE: 1.0542","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:42:07.049804Z","iopub.execute_input":"2025-01-01T08:42:07.050135Z","iopub.status.idle":"2025-01-01T08:42:07.062659Z","shell.execute_reply.started":"2025-01-01T08:42:07.050098Z","shell.execute_reply":"2025-01-01T08:42:07.061752Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef plot_rmsle_all_folds(cv_results, n_folds):\n    plt.figure(figsize=(20, 8))\n    \n    plt.subplot(1, 2, 1)\n    for fold in range(n_folds):\n        plt.plot(cv_results[fold].history['rmsle'], label=f'Fold {fold+1}')\n    plt.xlabel('Epoch')\n    plt.ylabel('RMSLE')\n    plt.title('Train RMSLE over Epochs for all Folds')\n    plt.legend(loc='upper right')\n    plt.grid(True)\n    \n    plt.subplot(1, 2, 2)\n    for fold in range(n_folds):\n        plt.plot(cv_results[fold].history['val_rmsle'], label=f'Fold {fold+1}')\n    plt.xlabel('Epoch')\n    plt.ylabel('RMSLE')\n    plt.title('Validation RMSLE over Epochs for all Folds')\n    plt.legend(loc='upper right')\n    plt.grid(True)\n    \n    plt.tight_layout()\n    plt.show()\n \nplot_rmsle_all_folds(cv_results, n_folds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:42:07.063743Z","iopub.execute_input":"2025-01-01T08:42:07.064016Z","iopub.status.idle":"2025-01-01T08:42:07.801895Z","shell.execute_reply.started":"2025-01-01T08:42:07.063991Z","shell.execute_reply":"2025-01-01T08:42:07.800823Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. Submission","metadata":{}},{"cell_type":"code","source":"# test predict\ntest_pred = model.predict(test_tf)\ntest_pred = pd.DataFrame(test_pred, columns=['Premium Amount'])\ntest_pred.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:42:07.803309Z","iopub.execute_input":"2025-01-01T08:42:07.803683Z","iopub.status.idle":"2025-01-01T08:42:54.902519Z","shell.execute_reply.started":"2025-01-01T08:42:07.803643Z","shell.execute_reply":"2025-01-01T08:42:54.901552Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({'id': test['id'], 'Premium Amount': test_pred['Premium Amount']})\nprint(submission.shape)\nsubmission.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:42:54.903961Z","iopub.execute_input":"2025-01-01T08:42:54.904649Z","iopub.status.idle":"2025-01-01T08:42:54.916154Z","shell.execute_reply.started":"2025-01-01T08:42:54.904603Z","shell.execute_reply":"2025-01-01T08:42:54.915276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:42:54.917092Z","iopub.execute_input":"2025-01-01T08:42:54.917336Z","iopub.status.idle":"2025-01-01T08:42:55.714876Z","shell.execute_reply.started":"2025-01-01T08:42:54.917311Z","shell.execute_reply":"2025-01-01T08:42:55.714149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-01T08:42:55.715926Z","iopub.execute_input":"2025-01-01T08:42:55.716209Z","iopub.status.idle":"2025-01-01T08:42:55.724559Z","shell.execute_reply.started":"2025-01-01T08:42:55.716179Z","shell.execute_reply":"2025-01-01T08:42:55.723643Z"}},"outputs":[],"execution_count":null}]}