{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:01.550051Z","iopub.execute_input":"2024-12-24T10:48:01.550454Z","iopub.status.idle":"2024-12-24T10:48:01.556924Z","shell.execute_reply.started":"2024-12-24T10:48:01.550412Z","shell.execute_reply":"2024-12-24T10:48:01.556063Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Solution: Verson 4 -- trying with tensorflow","metadata":{}},{"cell_type":"code","source":"train_path = '/kaggle/input/playground-series-s4e12/train.csv'\ntest_path = '/kaggle/input/playground-series-s4e12/test.csv'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:01.557938Z","iopub.execute_input":"2024-12-24T10:48:01.558132Z","iopub.status.idle":"2024-12-24T10:48:01.574914Z","shell.execute_reply.started":"2024-12-24T10:48:01.558114Z","shell.execute_reply":"2024-12-24T10:48:01.574243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a pandas training dataframe\ntrain_df = pd.read_csv(train_path)\ntrain_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:01.575899Z","iopub.execute_input":"2024-12-24T10:48:01.576163Z","iopub.status.idle":"2024-12-24T10:48:05.151382Z","shell.execute_reply.started":"2024-12-24T10:48:01.576142Z","shell.execute_reply":"2024-12-24T10:48:05.150463Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a pandas test dataframe\ntest_df = pd.read_csv(test_path)\ntest_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:05.152738Z","iopub.execute_input":"2024-12-24T10:48:05.153069Z","iopub.status.idle":"2024-12-24T10:48:07.470693Z","shell.execute_reply.started":"2024-12-24T10:48:05.153036Z","shell.execute_reply":"2024-12-24T10:48:07.469845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def rename_cols(my_dataframe):\n       my_dataframe.rename(columns={'Annual Income':'Income', 'Marital Status':'Married',\n       'Number of Dependents': 'Dependents', 'Education Level':'Education', 'Health Score':'HealthScore',\n       'Policy Type':'PolicyType', 'Previous Claims':'PrevClaims', 'Vehicle Age':'VehicleAge',\n       'Credit Score':'CreditScore', 'Insurance Duration':'InsDuration', 'Policy Start Date':'PolicyDate',\n       'Customer Feedback':'Feedback', 'Smoking Status':'Smokes', 'Exercise Frequency':'Exercises',\n       'Property Type':'Property'}, inplace=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:07.472157Z","iopub.execute_input":"2024-12-24T10:48:07.472428Z","iopub.status.idle":"2024-12-24T10:48:07.476886Z","shell.execute_reply.started":"2024-12-24T10:48:07.472406Z","shell.execute_reply":"2024-12-24T10:48:07.476008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rename_cols(train_df)\ntrain_df.rename(columns={'Premium Amount':'PremiumAmt'}, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:07.478033Z","iopub.execute_input":"2024-12-24T10:48:07.478258Z","iopub.status.idle":"2024-12-24T10:48:07.49242Z","shell.execute_reply.started":"2024-12-24T10:48:07.478232Z","shell.execute_reply":"2024-12-24T10:48:07.491738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rename_cols(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:07.493265Z","iopub.execute_input":"2024-12-24T10:48:07.493572Z","iopub.status.idle":"2024-12-24T10:48:07.506858Z","shell.execute_reply.started":"2024-12-24T10:48:07.493543Z","shell.execute_reply":"2024-12-24T10:48:07.506218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Convert start date to datetime and then integer\ndef change_datetime(my_dataframe):\n    my_dataframe['PolicyDate'] = pd.to_datetime(my_dataframe['PolicyDate'])\n    # train_df['PolicyMonth'] = pd.DatetimeIndex(train_df['PolicyDate']).month\n    # train_df['PolicyYear'] = pd.DatetimeIndex(train_df['PolicyDate']).year\n    my_dataframe['PolicyDate'] = my_dataframe['PolicyDate'].astype(np.int64)\n    return my_dataframe","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:07.507629Z","iopub.execute_input":"2024-12-24T10:48:07.507848Z","iopub.status.idle":"2024-12-24T10:48:07.519954Z","shell.execute_reply.started":"2024-12-24T10:48:07.507829Z","shell.execute_reply":"2024-12-24T10:48:07.519239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = change_datetime(train_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:07.520735Z","iopub.execute_input":"2024-12-24T10:48:07.52092Z","iopub.status.idle":"2024-12-24T10:48:07.902642Z","shell.execute_reply.started":"2024-12-24T10:48:07.520903Z","shell.execute_reply":"2024-12-24T10:48:07.901904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = change_datetime(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:07.904853Z","iopub.execute_input":"2024-12-24T10:48:07.905061Z","iopub.status.idle":"2024-12-24T10:48:08.151941Z","shell.execute_reply.started":"2024-12-24T10:48:07.905042Z","shell.execute_reply":"2024-12-24T10:48:08.151222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.153307Z","iopub.execute_input":"2024-12-24T10:48:08.153624Z","iopub.status.idle":"2024-12-24T10:48:08.157978Z","shell.execute_reply.started":"2024-12-24T10:48:08.153599Z","shell.execute_reply":"2024-12-24T10:48:08.157215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let us create a separate dataframe to view the null value counts\n\ndef show_nulls(my_dataframe, df_null):\n    col_list = my_dataframe.columns.values\n    data_types = my_dataframe.dtypes.values\n    df_null = pd.DataFrame(col_list, columns=['Cols'])\n    df_null['Dtype'] = data_types\n\n    null_values = my_dataframe.isnull().sum().values\n    df_null['Nulls'] = null_values\n\n    df_null = df_null[df_null['Nulls'] > 0]\n    return df_null\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.15881Z","iopub.execute_input":"2024-12-24T10:48:08.159042Z","iopub.status.idle":"2024-12-24T10:48:08.171964Z","shell.execute_reply.started":"2024-12-24T10:48:08.159022Z","shell.execute_reply":"2024-12-24T10:48:08.171312Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['Age'].median()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.172643Z","iopub.execute_input":"2024-12-24T10:48:08.172838Z","iopub.status.idle":"2024-12-24T10:48:08.215452Z","shell.execute_reply.started":"2024-12-24T10:48:08.172819Z","shell.execute_reply":"2024-12-24T10:48:08.214772Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['Age'].median()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.216224Z","iopub.execute_input":"2024-12-24T10:48:08.216557Z","iopub.status.idle":"2024-12-24T10:48:08.236838Z","shell.execute_reply.started":"2024-12-24T10:48:08.216525Z","shell.execute_reply":"2024-12-24T10:48:08.236148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"avg_age = train_df['Age'].median()\ntrain_df.loc[(train_df['Age'].isna()), 'Age'] = avg_age","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.237682Z","iopub.execute_input":"2024-12-24T10:48:08.237977Z","iopub.status.idle":"2024-12-24T10:48:08.266891Z","shell.execute_reply.started":"2024-12-24T10:48:08.237945Z","shell.execute_reply":"2024-12-24T10:48:08.266261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"avg_age = test_df['Age'].median()\ntest_df.loc[(test_df['Age'].isna()), 'Age'] = avg_age","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.267536Z","iopub.execute_input":"2024-12-24T10:48:08.26773Z","iopub.status.idle":"2024-12-24T10:48:08.28882Z","shell.execute_reply.started":"2024-12-24T10:48:08.267713Z","shell.execute_reply":"2024-12-24T10:48:08.288128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_age_grp(source_df):\n\n    age_bins = [18, 30, 40, 50, 75]\n    age_label_nos = {'18-30':1,'30-40':2,'40-50':3, '50-75':4}\n    source_df['AgeGroup'] = pd.cut(source_df['Age'], bins=age_bins, labels=age_label_nos.keys(), right=False)\n    \n    return source_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.289745Z","iopub.execute_input":"2024-12-24T10:48:08.290033Z","iopub.status.idle":"2024-12-24T10:48:08.294269Z","shell.execute_reply.started":"2024-12-24T10:48:08.290001Z","shell.execute_reply":"2024-12-24T10:48:08.293612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = calculate_age_grp(train_df)\ntrain_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.295032Z","iopub.execute_input":"2024-12-24T10:48:08.295285Z","iopub.status.idle":"2024-12-24T10:48:08.351137Z","shell.execute_reply.started":"2024-12-24T10:48:08.295265Z","shell.execute_reply":"2024-12-24T10:48:08.350307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = calculate_age_grp(test_df)\ntest_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.352021Z","iopub.execute_input":"2024-12-24T10:48:08.352296Z","iopub.status.idle":"2024-12-24T10:48:08.389866Z","shell.execute_reply.started":"2024-12-24T10:48:08.352267Z","shell.execute_reply":"2024-12-24T10:48:08.389008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# What is the average income per age group\ndef calc_avg_inc_per_age_grp(source_df):\n    income_per_age_grp = source_df.pivot_table(values='Income', index='AgeGroup', aggfunc='mean', observed=True)\n    return income_per_age_grp.to_dict()['Income']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.390632Z","iopub.execute_input":"2024-12-24T10:48:08.390838Z","iopub.status.idle":"2024-12-24T10:48:08.394762Z","shell.execute_reply.started":"2024-12-24T10:48:08.39082Z","shell.execute_reply":"2024-12-24T10:48:08.393867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# What is the average income per location\ndef calc_avg_inc_per_location(source_df):\n    income_per_loc = source_df.pivot_table(values='Income', index='Location', aggfunc='mean', observed=True)\n    return income_per_loc.to_dict()['Income']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.395471Z","iopub.execute_input":"2024-12-24T10:48:08.395683Z","iopub.status.idle":"2024-12-24T10:48:08.411494Z","shell.execute_reply.started":"2024-12-24T10:48:08.395664Z","shell.execute_reply":"2024-12-24T10:48:08.410863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# What is the average income per sex\ndef calc_avg_inc_per_sex(source_df):\n    income_per_sex = source_df.pivot_table(values='Income', index='Gender', aggfunc='mean', observed=True)\n    return income_per_sex.to_dict()['Income']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.41228Z","iopub.execute_input":"2024-12-24T10:48:08.412592Z","iopub.status.idle":"2024-12-24T10:48:08.428801Z","shell.execute_reply.started":"2024-12-24T10:48:08.412562Z","shell.execute_reply":"2024-12-24T10:48:08.428011Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# What is the average income per marital status\ndef calc_avg_inc_per_mstatus(source_df):\n    income_per_married = source_df.pivot_table(values='Income', index='Married', aggfunc='mean', observed=True)\n    return income_per_married.to_dict()['Income']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.429554Z","iopub.execute_input":"2024-12-24T10:48:08.429798Z","iopub.status.idle":"2024-12-24T10:48:08.44233Z","shell.execute_reply.started":"2024-12-24T10:48:08.429778Z","shell.execute_reply":"2024-12-24T10:48:08.441665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# What is the average income per education\ndef calc_avg_inc_edu(source_df):\n    income_per_edu = source_df.pivot_table(values='Income', index='Education', aggfunc='mean', observed=True)\n    return income_per_edu.to_dict()['Income']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.443145Z","iopub.execute_input":"2024-12-24T10:48:08.443432Z","iopub.status.idle":"2024-12-24T10:48:08.456379Z","shell.execute_reply.started":"2024-12-24T10:48:08.443403Z","shell.execute_reply":"2024-12-24T10:48:08.455719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# What is the average income per occupation\ndef calc_avg_inc_per_occ(source_df):\n    income_per_occ = source_df.pivot_table(values='Income', index='Occupation', aggfunc='mean', observed=True)\n    return income_per_occ.to_dict()['Income']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.459956Z","iopub.execute_input":"2024-12-24T10:48:08.460166Z","iopub.status.idle":"2024-12-24T10:48:08.472905Z","shell.execute_reply.started":"2024-12-24T10:48:08.460147Z","shell.execute_reply":"2024-12-24T10:48:08.472252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def massage_income(source_df):\n    # First get the dictionaries\n    age_dict = calc_avg_inc_per_age_grp(source_df)\n    location_dict = calc_avg_inc_per_location(source_df)\n    edu_dict = calc_avg_inc_edu(source_df)\n    marriage_dict =  calc_avg_inc_per_mstatus(source_df)\n    occ_dict =  calc_avg_inc_per_occ(source_df)\n    sex_dict = calc_avg_inc_per_sex(source_df)\n    \n    # There are a total of 44949 nulls in income\n    # Married and Occupation are the only cols that have null values along with income. \n    # Any rows where all three are null? I checked there are 304 \n    # For these, we'll apply the average of the groups for the relevant (age, gender, edu and location) of that row\n    temp_df = source_df[(source_df['Income'].isna()) & (source_df['Married'].isna()) & (source_df['Occupation'].isna())]\n    temp_df_indices = temp_df.index.values\n    \n    for k in range(len(temp_df)):\n\n        age_grp = temp_df.iloc[k]['AgeGroup']\n        loca = temp_df.iloc[k]['Location']\n        edu = temp_df.iloc[k]['Education']\n        sex = temp_df.iloc[k]['Gender']\n\n        whole_avg_income = (age_dict[age_grp] + location_dict[loca] + edu_dict[edu] + sex_dict[sex]) / 4\n        \n        source_df.loc[temp_df_indices[k], 'Income'] = whole_avg_income\n\n    # Any rows where income and married are null? I checked there are 652 \n    # For these, we'll apply the average of the groups for the relevant (age, gender, occupation, edu and location) of that row\n    temp_df = source_df[(source_df['Income'].isna()) & (source_df['Married'].isna()) & (~source_df['Occupation'].isna())]\n    temp_df_indices = temp_df.index.values\n    \n    for k in range(len(temp_df)):\n\n        age_grp = temp_df.iloc[k]['AgeGroup']\n        loca = temp_df.iloc[k]['Location']\n        edu = temp_df.iloc[k]['Education']\n        sex = temp_df.iloc[k]['Gender']\n        occ = temp_df.iloc[k]['Occupation']\n        \n        whole_avg_income = (age_dict[age_grp] + location_dict[loca] + edu_dict[edu] + sex_dict[sex] + occ_dict[occ]) / 5\n        \n        source_df.loc[temp_df_indices[k], 'Income'] = whole_avg_income\n    \n    # Any rows where income and occupation are null? I checked there are 12946 \n    # For these, we'll apply the average of the groups for the relevant (age, gender, married, edu and location) of that row\n    temp_df = source_df[(source_df['Income'].isna()) & (~source_df['Married'].isna()) & (source_df['Occupation'].isna())]\n    temp_df_indices = temp_df.index.values\n    \n    for k in range(len(temp_df)):\n\n        age_grp = temp_df.iloc[k]['AgeGroup']\n        loca = temp_df.iloc[k]['Location']\n        edu = temp_df.iloc[k]['Education']\n        sex = temp_df.iloc[k]['Gender']\n        marr = temp_df.iloc[k]['Married']\n        \n        whole_avg_income = (age_dict[age_grp] + location_dict[loca] + edu_dict[edu] + sex_dict[sex] + marriage_dict[marr]) / 5\n        \n        source_df.loc[temp_df_indices[k], 'Income'] = whole_avg_income\n    \n    # Rows with only the income as null? I checked there are 31047 \n    # For these, we'll apply the average of all the groups (age, gender, married, occupation, edu and location) of that row\n    temp_df = source_df[(source_df['Income'].isna())]\n    temp_df_indices = temp_df.index.values\n    \n    for k in range(len(temp_df)):\n\n        age_grp = temp_df.iloc[k]['AgeGroup']\n        loca = temp_df.iloc[k]['Location']\n        edu = temp_df.iloc[k]['Education']\n        sex = temp_df.iloc[k]['Gender']\n        marr = temp_df.iloc[k]['Married']\n        occ = temp_df.iloc[k]['Occupation']\n        \n        whole_avg_income = (age_dict[age_grp] + location_dict[loca] + edu_dict[edu] + sex_dict[sex] + \n                            occ_dict[occ] + marriage_dict[marr]) / 6\n        \n        source_df.loc[temp_df_indices[k], 'Income'] = whole_avg_income","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.474237Z","iopub.execute_input":"2024-12-24T10:48:08.474503Z","iopub.status.idle":"2024-12-24T10:48:08.487905Z","shell.execute_reply.started":"2024-12-24T10:48:08.47447Z","shell.execute_reply":"2024-12-24T10:48:08.487291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df_nulls = pd.DataFrame()\ntrain_df_nulls = show_nulls(train_df, train_df_nulls)\ntrain_df_nulls","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.488594Z","iopub.execute_input":"2024-12-24T10:48:08.488779Z","iopub.status.idle":"2024-12-24T10:48:08.974205Z","shell.execute_reply.started":"2024-12-24T10:48:08.488761Z","shell.execute_reply":"2024-12-24T10:48:08.973525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"massage_income(train_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:08.974938Z","iopub.execute_input":"2024-12-24T10:48:08.975147Z","iopub.status.idle":"2024-12-24T10:48:33.731339Z","shell.execute_reply.started":"2024-12-24T10:48:08.975128Z","shell.execute_reply":"2024-12-24T10:48:33.73006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df_nulls = pd.DataFrame()\ntest_df_nulls = show_nulls(test_df, test_df_nulls)\ntest_df_nulls","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:33.732195Z","iopub.execute_input":"2024-12-24T10:48:33.732433Z","iopub.status.idle":"2024-12-24T10:48:34.051543Z","shell.execute_reply.started":"2024-12-24T10:48:33.73241Z","shell.execute_reply":"2024-12-24T10:48:34.050697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"massage_income(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:34.052343Z","iopub.execute_input":"2024-12-24T10:48:34.052636Z","iopub.status.idle":"2024-12-24T10:48:50.449483Z","shell.execute_reply.started":"2024-12-24T10:48:34.052614Z","shell.execute_reply":"2024-12-24T10:48:50.448534Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Moving on to Credit Score. Credit score has a somewhat high correlation to income. A little bit with prev claims too but not so significant. Lets create an IncomeGroup and find the average Credit Score in each of those bins","metadata":{}},{"cell_type":"code","source":"def massage_creditscore(my_dataframe):\n\n    income_bins= [0, 1000, 5000, 15000, 50000, 100000, 500000]\n    income_label_nos = {'1K':1,'<5K':2,'<15K':3, '<50K':4, '<100K':5, '<500K':6}\n    my_dataframe['IncomeGroup'] = pd.cut(my_dataframe['Income'], bins=income_bins, labels=income_label_nos.keys(), right=False)\n\n    #grp_income = my_dataframe.pivot_table(values='PremiumAmt', index='IncomeGroup', aggfunc='mean', observed=True)\n\n    credit_score_perincome_grp = my_dataframe.pivot_table(values='CreditScore', index='IncomeGroup', aggfunc='mean', observed=True)\n\n    my_cs_dict = credit_score_perincome_grp.to_dict()['CreditScore']\n\n    for key in my_cs_dict:   \n        my_dataframe.loc[(my_dataframe['CreditScore'].isna()) & (my_dataframe['IncomeGroup'] == key), 'CreditScore'] = my_cs_dict[key]\n    \n    return my_dataframe\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.450406Z","iopub.execute_input":"2024-12-24T10:48:50.45068Z","iopub.status.idle":"2024-12-24T10:48:50.455981Z","shell.execute_reply.started":"2024-12-24T10:48:50.450657Z","shell.execute_reply":"2024-12-24T10:48:50.454989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = massage_creditscore(train_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.456709Z","iopub.execute_input":"2024-12-24T10:48:50.457013Z","iopub.status.idle":"2024-12-24T10:48:50.551688Z","shell.execute_reply.started":"2024-12-24T10:48:50.456978Z","shell.execute_reply":"2024-12-24T10:48:50.550699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = massage_creditscore(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.552673Z","iopub.execute_input":"2024-12-24T10:48:50.552993Z","iopub.status.idle":"2024-12-24T10:48:50.60822Z","shell.execute_reply.started":"2024-12-24T10:48:50.55296Z","shell.execute_reply":"2024-12-24T10:48:50.607584Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Define the pipeline to transform the other coumns","metadata":{}},{"cell_type":"code","source":"import sklearn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.608946Z","iopub.execute_input":"2024-12-24T10:48:50.60915Z","iopub.status.idle":"2024-12-24T10:48:50.613155Z","shell.execute_reply.started":"2024-12-24T10:48:50.609132Z","shell.execute_reply":"2024-12-24T10:48:50.612493Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.613936Z","iopub.execute_input":"2024-12-24T10:48:50.614289Z","iopub.status.idle":"2024-12-24T10:48:50.628763Z","shell.execute_reply.started":"2024-12-24T10:48:50.614249Z","shell.execute_reply":"2024-12-24T10:48:50.628085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_cols = ['Income', 'HealthScore', 'PrevClaims', 'CreditScore', 'InsDuration', 'PolicyDate']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.629469Z","iopub.execute_input":"2024-12-24T10:48:50.629692Z","iopub.status.idle":"2024-12-24T10:48:50.641552Z","shell.execute_reply.started":"2024-12-24T10:48:50.629673Z","shell.execute_reply":"2024-12-24T10:48:50.640771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_cols = [ 'Married', 'Feedback' ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.642341Z","iopub.execute_input":"2024-12-24T10:48:50.642655Z","iopub.status.idle":"2024-12-24T10:48:50.655266Z","shell.execute_reply.started":"2024-12-24T10:48:50.642625Z","shell.execute_reply":"2024-12-24T10:48:50.654498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Preprocessing for numerical data\n\nnumerical_transformer = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='mean')),\n    ('scaler', StandardScaler())\n])\n\n# Preprocessing for categorical data\ncategorical_transformer = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='most_frequent')),\n    ('onehot', OneHotEncoder())\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.655964Z","iopub.execute_input":"2024-12-24T10:48:50.656142Z","iopub.status.idle":"2024-12-24T10:48:50.670323Z","shell.execute_reply.started":"2024-12-24T10:48:50.656126Z","shell.execute_reply":"2024-12-24T10:48:50.669502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Combine preprocessing for numerical and categorical data\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', numerical_transformer, num_cols),\n        ('cat', categorical_transformer, cat_cols)\n    ])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.671025Z","iopub.execute_input":"2024-12-24T10:48:50.671201Z","iopub.status.idle":"2024-12-24T10:48:50.684357Z","shell.execute_reply.started":"2024-12-24T10:48:50.671185Z","shell.execute_reply":"2024-12-24T10:48:50.683684Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train_df[num_cols + cat_cols]\ny = train_df['PremiumAmt'].to_numpy().reshape(-1,1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.684951Z","iopub.execute_input":"2024-12-24T10:48:50.685138Z","iopub.status.idle":"2024-12-24T10:48:50.733197Z","shell.execute_reply.started":"2024-12-24T10:48:50.685121Z","shell.execute_reply":"2024-12-24T10:48:50.732531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.733919Z","iopub.execute_input":"2024-12-24T10:48:50.734151Z","iopub.status.idle":"2024-12-24T10:48:50.739127Z","shell.execute_reply.started":"2024-12-24T10:48:50.734131Z","shell.execute_reply":"2024-12-24T10:48:50.738472Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.20, random_state = 24)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.739952Z","iopub.execute_input":"2024-12-24T10:48:50.740201Z","iopub.status.idle":"2024-12-24T10:48:50.936272Z","shell.execute_reply.started":"2024-12-24T10:48:50.740182Z","shell.execute_reply":"2024-12-24T10:48:50.935611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_trn_scl = preprocessor.fit_transform(X_train)\nX_val_scl = preprocessor.transform(X_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:50.937012Z","iopub.execute_input":"2024-12-24T10:48:50.937214Z","iopub.status.idle":"2024-12-24T10:48:52.086944Z","shell.execute_reply.started":"2024-12-24T10:48:50.937195Z","shell.execute_reply":"2024-12-24T10:48:52.086254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_trn_scl.shape, X_val_scl.shape, y_val.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:52.087664Z","iopub.execute_input":"2024-12-24T10:48:52.087893Z","iopub.status.idle":"2024-12-24T10:48:52.092872Z","shell.execute_reply.started":"2024-12-24T10:48:52.087872Z","shell.execute_reply":"2024-12-24T10:48:52.092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.callbacks import EarlyStopping","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:52.093621Z","iopub.execute_input":"2024-12-24T10:48:52.09381Z","iopub.status.idle":"2024-12-24T10:48:52.106984Z","shell.execute_reply.started":"2024-12-24T10:48:52.093793Z","shell.execute_reply":"2024-12-24T10:48:52.106332Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Define the model","metadata":{}},{"cell_type":"code","source":"# Neural network\ntf.random.set_seed(24)\nmodel = tf.keras.models.Sequential([\n    tf.keras.layers.Dense(32, activation='relu',input_shape=(X_trn_scl.shape[1],),\n                          kernel_initializer='uniform'),\n    tf.keras.layers.Dropout(0.2),\n    \n    tf.keras.layers.Dense(64, activation='relu',\n                          kernel_initializer='uniform'),\n    tf.keras.layers.Dropout(0.2),\n    \n    tf.keras.layers.Dense(64, activation='relu',\n                          kernel_initializer='uniform'),\n    tf.keras.layers.Dropout(0.2),\n    \n    tf.keras.layers.Dense(32, activation='relu',\n                          kernel_initializer='uniform'),\n    tf.keras.layers.Dropout(0.2),\n    \n    tf.keras.layers.Dense(16, activation='relu',\n                          kernel_initializer='uniform'),\n    tf.keras.layers.Dropout(0.2),\n    \n    tf.keras.layers.Dense(8, activation='relu',\n                          kernel_initializer='uniform'),\n    \n    tf.keras.layers.Dense(1, 'linear', kernel_initializer='uniform')\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:52.107722Z","iopub.execute_input":"2024-12-24T10:48:52.107943Z","iopub.status.idle":"2024-12-24T10:48:52.69637Z","shell.execute_reply.started":"2024-12-24T10:48:52.107924Z","shell.execute_reply":"2024-12-24T10:48:52.695528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def my_rmsle(y_true, y_pred):\n    \"\"\"\n    Calculates the Root Mean Squared Logarithmic Error (RMSLE) using TensorFlow functions.\n\n    Args:\n        y_true: The ground truth values (TensorFlow tensor).\n        y_pred: The predicted values (TensorFlow tensor).\n\n    Returns:\n        The RMSLE (TensorFlow tensor).\n    \"\"\"\n    # Add 1 to both y_true and y_pred to avoid log(0) errors.\n    y_true = tf.cast(y_true, dtype=tf.float32)  # Ensure y_true is float32\n    y_pred = tf.cast(y_pred, dtype=tf.float32)  # Ensure y_pred is float32\n    y_true = tf.add(y_true, 1.0)\n    y_pred = tf.add(y_pred, 1.0)\n\n    # Calculate the log of y_true and y_pred.\n    log_true = tf.math.log(y_true)\n    log_pred = tf.math.log(y_pred)\n\n    # Calculate the squared difference between the logs.\n    squared_diff = tf.math.square(tf.math.subtract(log_pred, log_true))\n\n    # Calculate the mean of the squared differences.\n    mean_squared_diff = tf.reduce_mean(squared_diff)\n\n    # Calculate the square root of the mean squared difference.\n    rmsle = tf.math.sqrt(mean_squared_diff)\n\n    return rmsle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:52.697214Z","iopub.execute_input":"2024-12-24T10:48:52.697566Z","iopub.status.idle":"2024-12-24T10:48:52.702737Z","shell.execute_reply.started":"2024-12-24T10:48:52.697531Z","shell.execute_reply":"2024-12-24T10:48:52.701837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.compile(optimizer='adam', loss='mean_squared_error', metrics=[my_rmsle])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:52.703575Z","iopub.execute_input":"2024-12-24T10:48:52.703786Z","iopub.status.idle":"2024-12-24T10:48:52.727725Z","shell.execute_reply.started":"2024-12-24T10:48:52.703767Z","shell.execute_reply":"2024-12-24T10:48:52.727022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(model.summary())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:48:52.728463Z","iopub.execute_input":"2024-12-24T10:48:52.728673Z","iopub.status.idle":"2024-12-24T10:48:52.748129Z","shell.execute_reply.started":"2024-12-24T10:48:52.728655Z","shell.execute_reply":"2024-12-24T10:48:52.747519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monitor = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=5, verbose=2, restore_best_weights=True)\nhistory = model.fit(X_trn_scl, y_train, validation_data=(X_val_scl,y_val), callbacks=[monitor], verbose=2, epochs=20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T10:50:01.429185Z","iopub.execute_input":"2024-12-24T10:50:01.429583Z","iopub.status.idle":"2024-12-24T11:07:01.636161Z","shell.execute_reply.started":"2024-12-24T10:50:01.429554Z","shell.execute_reply":"2024-12-24T11:07:01.635452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper right')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T11:08:15.906839Z","iopub.execute_input":"2024-12-24T11:08:15.907198Z","iopub.status.idle":"2024-12-24T11:08:16.096014Z","shell.execute_reply.started":"2024-12-24T11:08:15.907169Z","shell.execute_reply":"2024-12-24T11:08:16.095168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test_df[num_cols + cat_cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T11:08:26.937904Z","iopub.execute_input":"2024-12-24T11:08:26.938188Z","iopub.status.idle":"2024-12-24T11:08:26.958555Z","shell.execute_reply.started":"2024-12-24T11:08:26.938166Z","shell.execute_reply":"2024-12-24T11:08:26.957837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test_scl = preprocessor.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T11:08:29.937254Z","iopub.execute_input":"2024-12-24T11:08:29.937592Z","iopub.status.idle":"2024-12-24T11:08:30.426523Z","shell.execute_reply.started":"2024-12-24T11:08:29.937562Z","shell.execute_reply":"2024-12-24T11:08:30.425787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test_scl.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T11:08:32.639088Z","iopub.execute_input":"2024-12-24T11:08:32.639374Z","iopub.status.idle":"2024-12-24T11:08:32.644191Z","shell.execute_reply.started":"2024-12-24T11:08:32.639352Z","shell.execute_reply":"2024-12-24T11:08:32.643432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insurance_premium_predictions = model.predict(X_test_scl)\ninsurance_premium_predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T11:08:36.626119Z","iopub.execute_input":"2024-12-24T11:08:36.626414Z","iopub.status.idle":"2024-12-24T11:09:18.245111Z","shell.execute_reply.started":"2024-12-24T11:08:36.62639Z","shell.execute_reply":"2024-12-24T11:09:18.2443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insurance_premium_predictions.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T11:11:42.275966Z","iopub.execute_input":"2024-12-24T11:11:42.276337Z","iopub.status.idle":"2024-12-24T11:11:42.281346Z","shell.execute_reply.started":"2024-12-24T11:11:42.276309Z","shell.execute_reply":"2024-12-24T11:11:42.280633Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_df = pd.DataFrame(test_df['id'], columns=['id'])\npred_df['Premium Amount'] = insurance_premium_predictions\npred_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T11:11:56.539948Z","iopub.execute_input":"2024-12-24T11:11:56.540237Z","iopub.status.idle":"2024-12-24T11:11:56.584294Z","shell.execute_reply.started":"2024-12-24T11:11:56.540214Z","shell.execute_reply":"2024-12-24T11:11:56.583544Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Save summary as csv file","metadata":{}},{"cell_type":"code","source":"summary_path = '/kaggle/working/summary.csv'\npred_df.to_csv(summary_path, sep=',', encoding='utf-8', index=False, header=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T11:12:10.250832Z","iopub.execute_input":"2024-12-24T11:12:10.25113Z","iopub.status.idle":"2024-12-24T11:12:11.195042Z","shell.execute_reply.started":"2024-12-24T11:12:10.251107Z","shell.execute_reply":"2024-12-24T11:12:11.194098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!wc -l summary.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T11:12:33.542924Z","iopub.execute_input":"2024-12-24T11:12:33.543261Z","iopub.status.idle":"2024-12-24T11:12:33.820177Z","shell.execute_reply.started":"2024-12-24T11:12:33.543226Z","shell.execute_reply":"2024-12-24T11:12:33.819241Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ------------------------------------ Thank you ------------------------------------------","metadata":{}}]}