{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":9178166,"sourceType":"datasetVersion","datasetId":5547076},{"sourceId":10085111,"sourceType":"datasetVersion","datasetId":6217867}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-27T07:55:22.178817Z","iopub.status.idle":"2025-05-27T07:55:22.179164Z","shell.execute_reply.started":"2025-05-27T07:55:22.178996Z","shell.execute_reply":"2025-05-27T07:55:22.179014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def check_columns(df, df_test):\n    extra_columns_df = [col for col in df.columns if col not in df_test.columns]\n    extra_columns_df_test = [col for col in df_test.columns if col not in df.columns]\n    if extra_columns_df or extra_columns_df_test:\n        print(\"Extra_columns_in_df\",extra_columns_df)\n        '''print(extra_columns_df_test)'''\n    else:\n        return None\n\n\ndef check_columns_dtype(df,df_test):\n    print(df.dtypes.value_counts())\n    print()\n    print(df_test.dtypes.value_counts())\n    print()\n\n\ndef to_predict(df_sample):\n    print(\"We have to predict the:\",df_sample.columns)\n    \n    \n    \ndef check_null_values(df,df_test):\n    print(\"NUll values in the df:\"), print(df.isnull().sum()),print()\n    print(\"Null values in the df_test:\"),print(df_test.isnull().sum()), print()\n    \ndef size_of_dataset(df,df_test):\n    print(f\"Size of the train_dataset:, {df.shape}\")\n    print(f\"Size of the train_dataset: {df_test.shape}\"),print()\n    print()\n\ndef unique_values(df, df_test):\n    print(\"Unique_values in df\")\n    for column in df.columns:\n        print(f\"Number of unique values in '{column}' : {df[column].nunique()}\")\n    print(\"Unique_values in df_test\"), print()\n    for column in df_test.columns:\n        print(f\"Number of unique values in '{column}' : {df_test[column].nunique()}\")\n\n                                         \n'''MAIN CDOE'''        \n\ndf= pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ndf_test=pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\ndf_sample =pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\n\ncheck_columns(df, df_test), print(),print()\ncheck_columns_dtype(df,df_test),print()\nto_predict(df_sample), print(), print()\ncheck_null_values(df,df_test)\nsize_of_dataset(df,df_test)\nunique_values(df,df_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T07:55:26.686544Z","iopub.execute_input":"2025-05-27T07:55:26.686959Z","iopub.status.idle":"2025-05-27T07:55:37.051361Z","shell.execute_reply.started":"2025-05-27T07:55:26.686925Z","shell.execute_reply":"2025-05-27T07:55:37.050187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"object_columns = df.select_dtypes(include=['object']).drop(columns=['Policy Start Date'])\n\n# Iterate and print unique values for object columns\nfor column in object_columns.columns:\n    unique_values = object_columns[column].unique()\n    print(f\"Unique values in column '{column}': {unique_values}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T07:49:15.840336Z","iopub.execute_input":"2025-05-27T07:49:15.84077Z","iopub.status.idle":"2025-05-27T07:49:16.744596Z","shell.execute_reply.started":"2025-05-27T07:49:15.840724Z","shell.execute_reply":"2025-05-27T07:49:16.743297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"NUll values in the df:\nid                           0\nAge                      18705\nGender                       0\nAnnual Income            44949\nMarital Status           18529\nNumber of Dependents    109672\nEducation Level              0\nOccupation              358075\nHealth Score             74076\nLocation                     0\nPolicy Type                  0\nPrevious Claims         364029\nVehicle Age                  6\nCredit Score            137882\nInsurance Duration           1\nPolicy Start Date            0\nCustomer Feedback        77824\nSmoking Status               0\nExercise Frequency           0\nProperty Type                0\nPremium Amount               0\ndtype: int64\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T07:49:16.746108Z","iopub.execute_input":"2025-05-27T07:49:16.746453Z","iopub.status.idle":"2025-05-27T07:49:16.755196Z","shell.execute_reply.started":"2025-05-27T07:49:16.746423Z","shell.execute_reply":"2025-05-27T07:49:16.753901Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport scipy.stats as stats\n\ndef comprehensive_outlier_visualization(df, column):\n    # Validate column exists\n    if column not in df.columns:\n        print(f\"Column '{column}' not found in DataFrame\")\n        return\n    \n    # Create a figure with multiple subplots\n    fig, axes = plt.subplots(2, 2, figsize=(16, 12))\n    fig.suptitle(f'Outlier Analysis Visualization for {column}', fontsize=16)\n    \n    # 1. Boxplot (Top Left)\n    sns.boxplot(x=df[column], ax=axes[0, 0])\n    axes[0, 0].set_title('Boxplot')\n    \n    # 2. Violin Plot (Bottom Left)\n    sns.violinplot(x=df[column], ax=axes[1, 0])\n    axes[1, 0].set_title('Violin Plot')\n    \n    # 3. Histogram with KDE (Top Right)\n    sns.histplot(df[column], kde=True, ax=axes[0, 1])\n    axes[0, 1].set_title('Histogram with KDE')\n    \n    # 4. Q-Q Plot (Bottom Right)\n    stats.probplot(df[column], plot=axes[1, 1])\n    axes[1, 1].set_title('Q-Q Plot')\n    \n    plt.tight_layout()\n    plt.show()\n\n# Alternative approach with error handling\ndef visualize_outliers(df, columns_list):\n    \"\"\"\n    Visualize outliers for multiple columns with error handling\n    \n    Parameters:\n    df (pandas.DataFrame): Input dataframe\n    columns_list (list): List of column names to analyze\n    \"\"\"\n    # Validate input\n    if not isinstance(columns_list, list):\n        columns_list = [columns_list]\n    \n    # Find valid columns\n    valid_columns = [col for col in columns_list if col in df.columns]\n    \n    # Check if any valid columns exist\n    if not valid_columns:\n        print(\"No valid columns found for visualization\")\n        return\n    \n    # Visualize each valid column\n    for column in valid_columns:\n        comprehensive_outlier_visualization(df, column)\n\n# Usage examples\ntry:\n    # Method 1: Single column\n    comprehensive_outlier_visualization(df, 'Previous Claims')\n    \n    # Method 2: Multiple columns with error handling\n    columns_to_check = ['Previous Claims', 'Health Score', 'Annual Income']\n    visualize_outliers(df, columns_to_check)\n\nexcept Exception as e:\n    print(f\"An error occurred: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T07:49:16.758118Z","iopub.execute_input":"2025-05-27T07:49:16.75858Z","iopub.status.idle":"2025-05-27T07:50:03.624349Z","shell.execute_reply.started":"2025-05-27T07:49:16.758549Z","shell.execute_reply":"2025-05-27T07:50:03.623298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nUnique values in column 'Gender': ['Female' 'Male']\nUnique values in column 'Marital Status': ['Married' 'Divorced' 'Single' nan]\nUnique values in column 'Education Level': [\"Bachelor's\" \"Master's\" 'High School' 'PhD']\nUnique values in column 'Occupation': ['Self-Employed' nan 'Employed' 'Unemployed']\nUnique values in column 'Location': ['Urban' 'Rural' 'Suburban']\nUnique values in column 'Policy Type': ['Premium' 'Comprehensive' 'Basic']\nUnique values in column 'Customer Feedback': ['Poor' 'Average' 'Good' nan]\nUnique values in column 'Smoking Status': ['No' 'Yes']\nUnique values in column 'Exercise Frequency': ['Weekly' 'Monthly' 'Daily' 'Rarely']\nUnique values in column 'Property Type': ['House' 'Apartment' 'Condo']\n\nNUll values in the df:\nid                           0\nAge                      18705\nGender                       0\nAnnual Income            44949\nMarital Status           18529\nNumber of Dependents    109672\nEducation Level              0\nOccupation              358075\nHealth Score             74076\nLocation                     0\nPolicy Type                  0\nPrevious Claims         364029\nVehicle Age                  6\nCredit Score            137882\nInsurance Duration           1\nPolicy Start Date            0\nCustomer Feedback        77824\nSmoking Status               0\nExercise Frequency           0\nProperty Type                0\nPremium Amount               0\ndtype: int64\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T07:50:03.625948Z","iopub.execute_input":"2025-05-27T07:50:03.626382Z","iopub.status.idle":"2025-05-27T07:50:03.635368Z","shell.execute_reply.started":"2025-05-27T07:50:03.626336Z","shell.execute_reply":"2025-05-27T07:50:03.634266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# \"\"\"Features with Missing Data: Number of Dependents, Credit Score\n\n# Features with Outliers: Previous Claims, Health Score, Annual Income\n\n# Features with Skewness: Annual Income, Health Score, Premium Amount\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T07:50:03.636663Z","iopub.execute_input":"2025-05-27T07:50:03.636988Z","iopub.status.idle":"2025-05-27T07:50:03.653674Z","shell.execute_reply.started":"2025-05-27T07:50:03.636958Z","shell.execute_reply":"2025-05-27T07:50:03.652636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ordinal encoding onn Gender\n\n# Fill missing values of Annual Income with median\n\n# skewnes: annual income, healthg score, premium ammount\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T07:50:03.655054Z","iopub.execute_input":"2025-05-27T07:50:03.655396Z","iopub.status.idle":"2025-05-27T07:50:03.668045Z","shell.execute_reply.started":"2025-05-27T07:50:03.655365Z","shell.execute_reply":"2025-05-27T07:50:03.666683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # if right skewed then median and if left skewed \n# Credit Score: mean\n# Health Score: mean\n# Annual Income: median\n# Number of dependents: mean\n# Age: mean\n\n# oridinal encoding of Gender: Female with 0, male with 1\n# oridinal encoding of Martial Status: Married with 2, Divorced with 1 , Single with 0\n# oridinal encoding of Education Level:High School with 0, Bachelors with 1, Masters with 2,  PhD with 3 \n# Encoding of Location: Rural with 0, sub-urban with 1 , urban with 2 \n# oridinal encoding of Smoking Status: No with 0 , yes with 1 \n\n# Exercise Frequency: ['Weekly' with 4, 'Monthly' with 1 ,  'Daily' with 28,  'Rarely' with 0.5\n# oridinal encoding of Property Type:['House' with 2 'Apartment' with 1 'Condo' with 0 ]\n# oridinal encoding of Policy Type:  ['Premium'  with 2  'Comprehensive' with 1 'Basic' with 0]\n# 'Occupation': ['Self-Employed' with 6 with ,nan with nan 'Employed' with 4 'Unemployed' with 2]\n# Customer Feedback': ['Poor' with 0  'Average' with 5  'Good' with 10, nan  with nan]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T07:50:03.669434Z","iopub.execute_input":"2025-05-27T07:50:03.669794Z","iopub.status.idle":"2025-05-27T07:50:03.679583Z","shell.execute_reply.started":"2025-05-27T07:50:03.669764Z","shell.execute_reply":"2025-05-27T07:50:03.678396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_skewness(df, features):\n    \"\"\"\n    Analyze skewness for specified features\n    \n    Parameters:\n    - df: Input DataFrame\n    - features: List of column names to analyze\n    \n    Returns:\n    - Dictionary with skewness details\n    \"\"\"\n    skewness_results = {}\n    \n    plt.figure(figsize=(20, 4 * len(features)))\n    \n    for idx, feature in enumerate(features, 1):\n        # Calculate Skewness\n        skewness = df[feature].skew()\n        \n        # Determine Skewness Type\n        if skewness > 1:\n            skew_type = \"Highly Right Skewed\"\n            recommended_metric = \"Median\"\n        elif skewness < -1:\n            skew_type = \"Highly Left Skewed\"\n            recommended_metric = \"Mode\"\n        elif skewness > 0.5:\n            skew_type = \"Moderately Right Skewed\"\n            recommended_metric = \"Median\"\n        elif skewness < -0.5:\n            skew_type = \"Moderately Left Skewed\"\n            recommended_metric = \"Mode\"\n        else:\n            skew_type = \"Approximately Symmetric\"\n            recommended_metric = \"Mean\"\n        \n        # Store Results\n        skewness_results[feature] = {\n            'skewness_value': skewness,\n            'skewness_type': skew_type,\n            'recommended_metric': recommended_metric\n        }\n        \n        # Visualization\n        plt.subplot(len(features), 2, 2*idx-1)\n        sns.histplot(df[feature], kde=True)\n        plt.title(f'{feature} Distribution\\nSkewness: {skewness:.4f}')\n        \n        # Box Plot\n        plt.subplot(len(features), 2, 2*idx)\n        sns.boxplot(x=df[feature])\n        plt.title(f'{feature} Boxplot')\n    \n    plt.tight_layout()\n    plt.show()\n    \n    return skewness_results\n\ndef print_skewness_report(skewness_results):\n    \"\"\"\n    Print detailed skewness report\n    \n    Parameters:\n    - skewness_results: Dictionary of skewness analysis results\n    \"\"\"\n    print(\"\\n--- Skewness Analysis Report ---\")\n    for feature, details in skewness_results.items():\n        print(f\"\\n{feature}:\")\n        print(f\"Skewness Value: {details['skewness_value']:.4f}\")\n        print(f\"Skewness Type: {details['skewness_type']}\")\n        print(f\"Recommended Metric: {details['recommended_metric']}\")\n\n# Main Execution\ndef main(df):\n    # Features to Analyze\n    features = [\n        'Credit Score', \n        'Health Score', \n        'Annual Income', \n        'Number of Dependents', \n        'Age',\n        'Premium Amount'\n    ]\n    \n    # Perform Skewness Analysis\n    skewness_results = analyze_skewness(df, features)\n    \n    # Print Detailed Report\n    print_skewness_report(skewness_results)\n    \n    return skewness_results\n\n# Usage\nskewness_results = main(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T07:50:03.680982Z","iopub.execute_input":"2025-05-27T07:50:03.68135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Define encoding mappings globally\nENCODING_MAPPINGS = {\n    'Gender': {\n        'Female': 0, \n        'Male': 1\n    },\n    'Marital Status': {\n        'Married': 2, \n        'Divorced': 1, \n        'Single': 0,\n        np.nan: np.nan\n    },\n    'Education Level': {\n        'High School': 0, \n        \"Bachelor's\": 1, \n        \"Master's\": 2, \n        'PhD': 3\n    },\n    'Location': {\n        'Rural': 0, \n        'Suburban': 1, \n        'Urban': 2\n    },\n    'Smoking Status': {\n        'No': 0, \n        'Yes': 1\n    },\n    'Exercise Frequency': {\n        'Rarely': 0.5, \n        'Monthly': 1, \n        'Weekly': 4, \n        'Daily': 28\n    },\n    'Property Type': {\n        'Condo': 0, \n        'Apartment': 1, \n        'House': 2\n    },\n    'Policy Type': {\n        'Basic': 0, \n        'Comprehensive': 1, \n        'Premium': 2\n    },\n    'Occupation': {\n        np.nan: np.nan,\n        'Unemployed': 2, \n        'Employed': 4, \n        'Self-Employed': 6\n    },\n    'Customer Feedback': {\n        'Poor': 0, \n        'Average': 5, \n        'Good': 10,\n        'No Feedback': np.nan,\n        np.nan: np.nan\n    }\n}\n\ndef ordinal_encoding(df):\n    \"\"\"\n    Perform ordinal encoding on categorical columns\n    \n    Parameters:\n    - df: Input DataFrame\n    \n    Returns:\n    - DataFrame with ordinal encoded columns\n    \"\"\"\n    # Deep copy to avoid modifying original DataFrame\n    encoded_df = df.copy()\n    \n    # Apply Ordinal Encoding\n    for column, mapping in ENCODING_MAPPINGS.items():\n        if column in df.columns:\n            encoded_df[column] = df[column].map(mapping)\n    \n    return encoded_df\n\ndef validate_encoding(df, encoded_df):\n    \"\"\"\n    Validate the encoding process\n    \n    Parameters:\n    - df: Original DataFrame\n    - encoded_df: Encoded DataFrame\n    \n    Returns:\n    - Detailed encoding validation report\n    \"\"\"\n    print(\"\\n--- Encoding Validation Report ---\")\n    \n    for column in encoded_df.columns:\n        if column in ENCODING_MAPPINGS:\n            print(f\"\\n{column} Encoding:\")\n            \n            # Original Unique Values\n            print(\"Original Unique Values:\")\n            print(df[column].unique())\n            \n            # Encoded Unique Values\n            print(\"\\nEncoded Unique Values:\")\n            print(encoded_df[column].unique())\n            \n            # Mapping Verification\n            mapping_verification = df[column].value_counts().to_frame()\n            mapping_verification['Encoded Value'] = mapping_verification.index.map(\n                ENCODING_MAPPINGS.get(column, {})\n            )\n            \n            print(\"\\nMapping Verification:\")\n            print(mapping_verification)\n            print(\"-\" * 50)\n    \n    return encoded_df\n\ndef main(df):\n    \"\"\"\n    Main function to perform ordinal encoding\n    \n    Parameters:\n    - df: Input DataFrame\n    \n    Returns:\n    - Encoded DataFrame with validation report\n    \"\"\"\n    # Perform Ordinal Encoding\n    encoded_df = ordinal_encoding(df)\n    \n    # Validate Encoding\n    validated_df = validate_encoding(df, encoded_df)\n    \n    return validated_df\n\n# Usage\nencoded_df = main(df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ndef analyze_skewness(df, features):\n    \"\"\"\n    Analyze skewness for specified encoded features\n    \n    Parameters:\n    - df: Input DataFrame\n    - features: List of column names to analyze\n    \n    Returns:\n    - Dictionary with skewness details\n    \"\"\"\n    # Ensure necessary imports\n    import scipy.stats as stats\n    \n    # Initialize results dictionary\n    skewness_results = {}\n    \n    # Create a figure with appropriate size\n    plt.figure(figsize=(20, 4 * len(features)))\n    \n    for idx, feature in enumerate(features, 1):\n        # Skip NaN values\n        feature_data = df[feature].dropna()\n        \n        # Calculate Skewness\n        try:\n            # Calculate skewness\n            skewness = feature_data.skew()\n            \n            # Shapiro-Wilk test for normality\n            _, p_value = stats.shapiro(feature_data)\n            \n            # Determine Skewness Type\n            if abs(skewness) > 1:\n                skew_type = \"Highly Skewed\"\n            elif abs(skewness) > 0.5:\n                skew_type = \"Moderately Skewed\"\n            else:\n                skew_type = \"Approximately Symmetric\"\n            \n            # Determine Transformation Recommendation\n            if skewness > 1:\n                recommended_transform = \"Log or Square Root\"\n            elif skewness < -1:\n                recommended_transform = \"Exponential\"\n            else:\n                recommended_transform = \"No Transform Needed\"\n            \n            # Store Results\n            skewness_results[feature] = {\n                'skewness_value': skewness,\n                'skewness_type': skew_type,\n                'shapiro_p_value': p_value,\n                'is_normal': p_value > 0.05,\n                'recommended_transform': recommended_transform\n            }\n            \n            # Visualization\n            plt.subplot(len(features), 2, 2*idx-1)\n            sns.histplot(feature_data, kde=True)\n            plt.title(f'{feature} Distribution\\nSkewness: {skewness:.4f}')\n            \n            # Box Plot\n            plt.subplot(len(features), 2, 2*idx)\n            sns.boxplot(x=feature_data)\n            plt.title(f'{feature} Boxplot')\n        \n        except Exception as e:\n            print(f\"Error processing {feature}: {e}\")\n            skewness_results[feature] = {\n                'skewness_value': None,\n                'error': str(e)\n            }\n    \n    plt.tight_layout()\n    plt.show()\n    \n    return skewness_results\n\ndef print_skewness_report(skewness_results):\n    \"\"\"\n    Print detailed skewness report\n    \n    Parameters:\n    - skewness_results: Dictionary of skewness analysis results\n    \"\"\"\n    print(\"\\n--- Encoded Data Skewness Analysis Report ---\")\n    for feature, details in skewness_results.items():\n        print(f\"\\n{feature}:\")\n        \n        # Handle potential errors\n        if 'error' in details:\n            print(f\"Error in analysis: {details['error']}\")\n            continue\n        \n        # Print detailed information\n        print(f\"Skewness Value: {details['skewness_value']:.4f}\")\n        print(f\"Skewness Type: {details['skewness_type']}\")\n        print(f\"Shapiro-Wilk P-Value: {details.get('shapiro_p_value', 'N/A'):.4f}\")\n        print(f\"Normally Distributed: {details.get('is_normal', 'N/A')}\")\n        print(f\"Recommended Transformation: {details['recommended_transform']}\")\n\ndef main(df):\n    \"\"\"\n    Main function to perform skewness analysis on encoded data\n    \n    Parameters:\n    - df: Input DataFrame (already encoded)\n    \n    Returns:\n    - Skewness analysis results\n    \"\"\"\n    # Features to Analyze (use the same list from encoding)\n    features = [\n        'Gender',\n        'Marital Status',\n        'Education Level',\n        'Location',\n        'Smoking Status',\n        'Exercise Frequency', \n        'Property Type',\n        'Policy Type',\n        'Occupation',\n        'Customer Feedback'\n    ]\n    \n    # Perform Skewness Analysis\n    skewness_results = analyze_skewness(df, features)\n    \n    # Print Detailed Report\n    print_skewness_report(skewness_results)\n    \n    return skewness_results\n\n# Usage\n# Assuming df_encoded is your encoded DataFrame\nskewness_results = main(encoded_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filling missing values in the encoded_df dataset\nencoded_df['Credit Score'].fillna(encoded_df['Credit Score'].mean(), inplace=True)\nencoded_df['Health Score'].fillna(encoded_df['Health Score'].mean(), inplace=True)\nencoded_df['Annual Income'].fillna(encoded_df['Annual Income'].median(), inplace=True)\nencoded_df['Number of Dependents'].fillna(encoded_df['Number of Dependents'].mean(), inplace=True)\nencoded_df['Age'].fillna(encoded_df['Age'].mean(), inplace=True)\nencoded_df['Vehicle Age'].fillna(encoded_df['Age'].mean(),inplace = True)\nencoded_df['Insurance Duration'].fillna(encoded_df['Insurance Duration'].mean(),inplace = True)\nencoded_df['Previous Claims'].fillna(encoded_df['Previous Claims'].median(), inplace=True)\nencoded_df['Marital Status'].fillna(encoded_df['Marital Status'].mode()[0], inplace=True)\nencoded_df['Occupation'].fillna(encoded_df['Occupation'].mode()[0], inplace=True)\nencoded_df['Customer Feedback'].fillna(encoded_df['Customer Feedback'].mode()[0], inplace=True )\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nMarital Status, Occupation, Previous Claims, Customer Feedback\nUnique values in column 'Occupation': ['Self-Employed' nan 'Employed' 'Unemployed']\nUnique values in column 'Marital Status': ['Married' 'Divorced' 'Single' nan]\n\nUnique values in column 'Education Level': [\"Bachelor's\" \"Master's\" 'High School' 'PhD']\n\nUnique values in column 'Location': ['Urban' 'Rural' 'Suburban']\nUnique values in column 'Policy Type': ['Premium' 'Comprehensive' 'Basic']\nUnique values in column 'Customer Feedback': ['Poor' 'Average' 'Good' nan]\nUnique values in column 'Smoking Status': ['No' 'Yes']\nUnique values in column 'Exercise Frequency': ['Weekly' 'Monthly' 'Daily' 'Rarely']\nUnique values in column 'Property Type': ['House' 'Apartment' 'Condo']\n\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ndef analyze_skewness(df, features):\n    \"\"\"\n    Analyze skewness for specified encoded features\n    \n    Parameters:\n    - df: Input DataFrame\n    - features: List of column names to analyze\n    \n    Returns:\n    - Dictionary with skewness details\n    \"\"\"\n    # Ensure necessary imports\n    import scipy.stats as stats\n    \n    # Initialize results dictionary\n    skewness_results = {}\n    \n    # Create a figure with appropriate size\n    plt.figure(figsize=(20, 4 * len(features)))\n    \n    for idx, feature in enumerate(features, 1):\n        # Skip NaN values\n        feature_data = df[feature].dropna()\n        \n        # Calculate Skewness\n        try:\n            # Calculate skewness\n            skewness = feature_data.skew()\n            \n            # Shapiro-Wilk test for normality\n            _, p_value = stats.shapiro(feature_data)\n            \n            # Determine Skewness Type\n            if abs(skewness) > 1:\n                skew_type = \"Highly Skewed\"\n            elif abs(skewness) > 0.5:\n                skew_type = \"Moderately Skewed\"\n            else:\n                skew_type = \"Approximately Symmetric\"\n            \n            # Determine Transformation Recommendation\n            if skewness > 1:\n                recommended_transform = \"Log or Square Root\"\n            elif skewness < -1:\n                recommended_transform = \"Exponential\"\n            else:\n                recommended_transform = \"No Transform Needed\"\n            \n            # Store Results\n            skewness_results[feature] = {\n                'skewness_value': skewness,\n                'skewness_type': skew_type,\n                'shapiro_p_value': p_value,\n                'is_normal': p_value > 0.05,\n                'recommended_transform': recommended_transform\n            }\n            \n            # Visualization\n            plt.subplot(len(features), 2, 2*idx-1)\n            sns.histplot(feature_data, kde=True)\n            plt.title(f'{feature} Distribution\\nSkewness: {skewness:.4f}')\n            \n            # Box Plot\n            plt.subplot(len(features), 2, 2*idx)\n            sns.boxplot(x=feature_data)\n            plt.title(f'{feature} Boxplot')\n        \n        except Exception as e:\n            print(f\"Error processing {feature}: {e}\")\n            skewness_results[feature] = {\n                'skewness_value': None,\n                'error': str(e)\n            }\n    \n    plt.tight_layout()\n    plt.show()\n    \n    return skewness_results\n\ndef print_skewness_report(skewness_results):\n    \"\"\"\n    Print detailed skewness report\n    \n    Parameters:\n    - skewness_results: Dictionary of skewness analysis results\n    \"\"\"\n    print(\"\\n--- Encoded Data Skewness Analysis Report ---\")\n    for feature, details in skewness_results.items():\n        print(f\"\\n{feature}:\")\n        \n        # Handle potential errors\n        if 'error' in details:\n            print(f\"Error in analysis: {details['error']}\")\n            continue\n        \n        # Print detailed information\n        print(f\"Skewness Value: {details['skewness_value']:.4f}\")\n        print(f\"Skewness Type: {details['skewness_type']}\")\n        print(f\"Shapiro-Wilk P-Value: {details.get('shapiro_p_value', 'N/A'):.4f}\")\n        print(f\"Normally Distributed: {details.get('is_normal', 'N/A')}\")\n        print(f\"Recommended Transformation: {details['recommended_transform']}\")\n\ndef main(df):\n    \"\"\"\n    Main function to perform skewness analysis on encoded data\n    \n    Parameters:\n    - df: Input DataFrame (already encoded)\n    \n    Returns:\n    - Skewness analysis results\n    \"\"\"\n    # Features to Analyze (use the same list from encoding)\n    features = ['Previous Claims','Marital Status', 'Occupation','Customer Feedback']\n    \n    # Perform Skewness Analysis\n    skewness_results = analyze_skewness(df, features)\n    \n    # Print Detailed Report\n    print_skewness_report(skewness_results)\n    \n    return skewness_results\n\n# Usage\n# Assuming df_encoded is your encoded DataFrame\nskewness_results = main(encoded_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoded_df['Policy Start Date'].head(10)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom datetime import datetime\n\ndef analyze_date_column(df, column_name):\n    \"\"\"\n    Analyze a date column to find oldest, newest, and other key statistics\n    \n    Parameters:\n    - df: DataFrame containing the date column\n    - column_name: Name of the date column\n    \n    Returns:\n    - Dictionary with date analysis results\n    \"\"\"\n    # Ensure the column is datetime\n    df[column_name] = pd.to_datetime(df[column_name])\n    \n    # Date Analysis\n    date_analysis = {\n        'oldest_date': df[column_name].min(),\n        'newest_date': df[column_name].max(),\n        'date_range': df[column_name].max() - df[column_name].min(),\n        'total_unique_dates': df[column_name].nunique(),\n        'date_distribution': {\n            'year_distribution': df[column_name].dt.year.value_counts(),\n            'month_distribution': df[column_name].dt.month.value_counts(),\n        }\n    }\n    \n    # Detailed Reporting\n    print(\"\\n--- Date Column Analysis ---\")\n    print(f\"Column Name: {column_name}\")\n    print(f\"Oldest Date: {date_analysis['oldest_date']}\")\n    print(f\"Newest Date: {date_analysis['newest_date']}\")\n    print(f\"Total Date Range: {date_analysis['date_range']}\")\n    print(f\"Total Unique Dates: {date_analysis['total_unique_dates']}\")\n    \n    # Year Distribution\n    print(\"\\nYear Distribution:\")\n    print(date_analysis['date_distribution']['year_distribution'])\n    \n    # Month Distribution\n    print(\"\\nMonth Distribution:\")\n    print(date_analysis['date_distribution']['month_distribution'])\n    \n    # Additional Visualizations\n    plt.figure(figsize=(15, 5))\n    \n    # Year Distribution Plot\n    plt.subplot(1, 2, 1)\n    date_analysis['date_distribution']['year_distribution'].plot(kind='bar')\n    plt.title('Date Distribution by Year')\n    plt.xlabel('Year')\n    plt.ylabel('Count')\n    \n    # Month Distribution Plot\n    plt.subplot(1, 2, 2)\n    date_analysis['date_distribution']['month_distribution'].plot(kind='bar')\n    plt.title('Date Distribution by Month')\n    plt.xlabel('Month')\n    plt.ylabel('Count')\n    \n    plt.tight_layout()\n    plt.show()\n    \n    return date_analysis\n\n# Additional helper functions\ndef calculate_policy_duration(df, start_date_column):\n    \"\"\"\n    Calculate policy duration from start date to current date\n    \n    Parameters:\n    - df: DataFrame\n    - start_date_column: Name of the start date column\n    \n    Returns:\n    - Series of policy durations\n    \"\"\"\n    current_date = pd.Timestamp.now()\n    df[start_date_column] = pd.to_datetime(df[start_date_column])\n    \n    # Calculate duration\n    policy_duration = current_date - df[start_date_column]\n    \n    # Convert to years, months, days\n    years = policy_duration.dt.days // 365\n    months = (policy_duration.dt.days % 365) // 30\n    days = (policy_duration.dt.days % 365) % 30\n    \n    # Create summary\n    duration_summary = pd.DataFrame({\n        'Years': years,\n        'Months': months,\n        'Days': days\n    })\n    \n    print(\"\\n--- Policy Duration Analysis ---\")\n    print(\"Policy Duration Statistics:\")\n    print(duration_summary.describe())\n    \n    return duration_summary\n\n# Main execution function\ndef main(df, column_name):\n    \"\"\"\n    Main function to perform date column analysis\n    \n    Parameters:\n    - df: DataFrame\n    - column_name: Name of the date column\n    \n    Returns:\n    - Date analysis results\n    \"\"\"\n    # Analyze Date Column\n    date_analysis = analyze_date_column(df, column_name)\n    \n    # Calculate Policy Duration\n    policy_duration = calculate_policy_duration(df, column_name)\n    \n    return {\n        'date_analysis': date_analysis,\n        'policy_duration': policy_duration\n    }\n\n# Usage\nresults = main(encoded_df, 'Policy Start Date')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndef create_date_features(df_date_data):\n    \"\"\"\n    Comprehensive date feature extraction with robust handling\n    \n    Parameters:\n    - df_date_data: DataFrame or Series with date information\n    \n    Returns:\n    - DataFrame with extracted date features\n    \"\"\"\n    # Ensure we're working with a DataFrame or Series\n    if not isinstance(df_date_data, (pd.DataFrame, pd.Series)):\n        raise ValueError(\"Input must be a pandas DataFrame or Series\")\n    \n    # Convert to DataFrame if it's a Series\n    if isinstance(df_date_data, pd.Series):\n        df_date_data = df_date_data.to_frame(name='Policy_Start_Date')\n    \n    # Ensure date column exists\n    if 'Policy_Start_Date' not in df_date_data.columns:\n        df_date_data = df_date_data.rename(columns={df_date_data.columns[0]: 'Policy_Start_Date'})\n    \n    # Convert to datetime with error handling\n    try:\n        df_date_data['Policy_Start_Date'] = pd.to_datetime(\n            df_date_data['Policy_Start_Date'], \n            errors='coerce'  # Convert invalid dates to NaT\n        )\n    except Exception as e:\n        print(f\"Error converting to datetime: {e}\")\n        return df_date_data\n    \n    # Remove rows with invalid dates if needed\n    df_date_data = df_date_data.dropna(subset=['Policy_Start_Date'])\n    \n    # Date Extraction Features\n    date_features = {\n        'Policy_Start_Year': df_date_data['Policy_Start_Date'].dt.year,\n        'Policy_Start_Month': df_date_data['Policy_Start_Date'].dt.month,\n        'Policy_Start_Day': df_date_data['Policy_Start_Date'].dt.day,\n        'Policy_Start_Quarter': df_date_data['Policy_Start_Date'].dt.quarter,\n        'Policy_Start_Day_of_Week': df_date_data['Policy_Start_Date'].dt.dayofweek,\n        'Policy_Start_Is_Weekend': df_date_data['Policy_Start_Date'].dt.dayofweek.isin([5,6]).astype(int)\n    }\n    \n    # Add extracted features to DataFrame\n    for feature_name, feature_data in date_features.items():\n        df_date_data[feature_name] = feature_data\n    \n    # Policy Age Calculation\n    reference_date = pd.Timestamp.now()\n    df_date_data['Policy_Age_Days'] = (reference_date - df_date_data['Policy_Start_Date']).dt.days\n    df_date_data['Policy_Age_Years'] = df_date_data['Policy_Age_Days'] / 365.25\n    \n    # Seasonality Features\n    season_map = {\n        1: 'Winter', 2: 'Winter', 3: 'Spring', \n        4: 'Spring', 5: 'Spring', 6: 'Summer', \n        7: 'Summer', 8: 'Summer', 9: 'Autumn', \n        10: 'Autumn', 11: 'Autumn', 12: 'Winter'\n    }\n    df_date_data['Policy_Start_Season'] = df_date_data['Policy_Start_Month'].map(season_map)\n    \n    # Advanced Date Features\n    df_date_data['Is_Leap_Year'] = df_date_data['Policy_Start_Date'].dt.is_leap_year\n    df_date_data['Days_In_Month'] = df_date_data['Policy_Start_Date'].dt.days_in_month\n    \n    # Date Range Percentile\n    df_date_data['Date_Range_Percentile'] = (\n        (df_date_data['Policy_Start_Date'] - df_date_data['Policy_Start_Date'].min()) / \n        (df_date_data['Policy_Start_Date'].max() - df_date_data['Policy_Start_Date'].min())\n    ) * 100\n    \n    # Comprehensive Date Analysis\n    print(\"\\n--- Comprehensive Date Feature Analysis ---\")\n    print(\"\\nDate Range:\")\n    print(f\"Earliest Date: {df_date_data['Policy_Start_Date'].min()}\")\n    print(f\"Latest Date: {df_date_data['Policy_Start_Date'].max()}\")\n    \n    print(\"\\nFeature Distribution:\")\n    for feature in ['Policy_Start_Year', 'Policy_Start_Month', 'Policy_Start_Quarter', 'Policy_Start_Season']:\n        print(f\"\\n{feature} Distribution:\")\n        print(df_date_data[feature].value_counts())\n    \n    # Optional Visualization\n    try:\n        import matplotlib.pyplot as plt\n        \n        plt.figure(figsize=(15, 10))\n        \n        # Yearly Distribution\n        plt.subplot(2, 2, 1)\n        df_date_data['Policy_Start_Year'].value_counts().sort_index().plot(kind='bar')\n        plt.title('Yearly Distribution')\n        plt.xlabel('Year')\n        plt.ylabel('Count')\n        \n        # Monthly Distribution\n        plt.subplot(2, 2, 2)\n        df_date_data['Policy_Start_Month'].value_counts().sort_index().plot(kind='bar')\n        plt.title('Monthly Distribution')\n        plt.xlabel('Month')\n        plt.ylabel('Count')\n        \n        # Quarter Distribution\n        plt.subplot(2, 2, 3)\n        df_date_data['Policy_Start_Quarter'].value_counts().sort_index().plot(kind='bar')\n        plt.title('Quarterly Distribution')\n        plt.xlabel('Quarter')\n        plt.ylabel('Count')\n        \n        # Season Distribution\n        plt.subplot(2, 2, 4)\n        df_date_data['Policy_Start_Season'].value_counts().plot(kind='pie', autopct='%1.1f%%')\n        plt.title('Seasonal Distribution')\n        \n        plt.tight_layout()\n        plt.show()\n    except Exception as viz_error:\n        print(f\"Visualization error: {viz_error}\")\n    \n    return df_date_data\n\n# Usage Examples\n# Method 1: With DataFrame\n# df_with_date_features = create_date_features(df)\n\n# Method 2: With Series\ndf_with_date_features = create_date_features(df['Policy Start Date'])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_with_date_features.sample(20)\n\n# drop these column Policy_Start_date, Policy_Start_Quarter, Policy_Start_Day_of_Week, Policy_Start_Is_Weekend, Policy_Start_Season, Is_Leap_Year, Days_In_Month\tDate_Range_Percentile from df_wtih_date_features","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sample 20 rows\nsampled_df = df_with_date_features.sample(20)\n\n# Drop specified columns\ncolumns_to_drop = [\n    'Policy_Start_Date', \n    'Policy_Start_Quarter', \n    'Policy_Start_Day_of_Week', \n    'Policy_Start_Is_Weekend', \n    'Policy_Start_Season', \n    'Is_Leap_Year', \n    'Days_In_Month', \n    'Date_Range_Percentile',\n    'Policy_Start_Year'\n]\n\ndf_cleaned = df_with_date_features.drop(columns=columns_to_drop)\n\n\ndf_cleaned.sample(20)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Simple Concatenation\nfinal_dataframe = pd.concat([encoded_df, df_cleaned], axis=1)\n\n# Remove duplicate columns\nfinal_dataframe = final_dataframe.loc[:, ~final_dataframe.columns.duplicated()]\nfinal_dataframe.drop('Policy Start Date', axis=1, inplace=True)\n\n\n# Quick verification\nprint(final_dataframe.info())\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_dataframe.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_dataframe.to_csv(\"final_dataframe.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"NUll values in the df:\"), print(final_dataframe.isnull().sum()),print()\nprint(final_dataframe.dtypes.value_counts())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# c = final_dataframe.columns\n# for i in c:\n#     if final_dataframe[i].dtypes == 'object':\n#         print(i)\n# # output = Customer feedback\n# # final_dataframe['Customer Feedback'].head(20)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Detailed Type Investigation\n# print(final_dataframe['Customer Feedback'].dtype)\n# print(final_dataframe['Customer Feedback'].unique())\n\n# # Explicit Type Conversion\n# final_dataframe['Customer Feedback'] = pd.to_numeric(final_dataframe['Customer Feedback'], errors='coerce')\n\n# # Verify Type\n# print(final_dataframe['Customer Feedback'].dtype)\n\n# # Alternative Approach\n# def convert_to_numeric(series):\n#     try:\n#         return pd.to_numeric(series)\n#     except:\n#         return series\n\n# # Apply to entire dataframe\n# final_dataframe = final_dataframe.apply(convert_to_numeric)\n\n# # Categorical Column Detection\n# categorical_cols = final_dataframe.select_dtypes(include=['object']).columns\n# print(\"Categorical Columns:\", list(categorical_cols))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_dataframe['Customer Feedback'].nunique()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#Handling skewness\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import PowerTransformer, StandardScaler, RobustScaler\n\ndef transform_features(final_dataframe):\n    \"\"\"\n    Perform feature transformations based on skewness characteristics\n    \n    Parameters:\n    -----------\n    final_dataframe : pandas.DataFrame\n        Input DataFrame containing the features to be transformed\n    \n    Returns:\n    --------\n    pandas.DataFrame\n        DataFrame with transformed features\n    \"\"\"\n    # Create a copy of the dataframe to avoid modifying the original\n    transformed_df = final_dataframe.copy()\n    \n    # 1. Credit Score (Approximately Symmetric)\n    # Standard scaling is appropriate for approximately symmetric distributions\n    transformed_df['Credit_Score_Scaled'] = StandardScaler().fit_transform(\n        transformed_df[['Credit Score']]\n    )\n    \n    # 2. Health Score (Approximately Symmetric)\n    # Standard scaling is appropriate for approximately symmetric distributions\n    transformed_df['Health_Score_Scaled'] = StandardScaler().fit_transform(\n        transformed_df[['Health Score']]\n    )\n    \n    # 3. Annual Income (Highly Right Skewed)\n    # Log transformation or Yeo-Johnson transformation can help\n    pt = PowerTransformer(method='yeo-johnson')\n    transformed_df['Annual_Income_Transformed'] = pt.fit_transform(\n        transformed_df[['Annual Income']]\n    )\n    \n    # 4. Premium Amount (Highly Right Skewed)\n    # Log transformation or Yeo-Johnson transformation can help\n    pt_premium = PowerTransformer(method='yeo-johnson')\n    transformed_df['Premium_Amount_Transformed'] = pt_premium.fit_transform(\n        transformed_df[['Premium Amount']]\n    )\n    \n    # Additional options for comparison\n    # Log transformation (alternative approach)\n    transformed_df['Annual_Income_Log'] = np.log1p(transformed_df['Annual Income'])\n    transformed_df['Premium_Amount_Log'] = np.log1p(transformed_df['Premium Amount'])\n    \n    # Robust scaling for comparison (less sensitive to outliers)\n    transformed_df['Annual_Income_Robust_Scaled'] = RobustScaler().fit_transform(\n        transformed_df[['Annual Income']]\n    )\n    \n    return transformed_df\n\n# Example usage\ndef main():\n    # Assuming final_dataframe is already loaded\n    transformed_data = transform_features(final_dataframe)\n    \n    # Visualization of transformations (optional)\n    import matplotlib.pyplot as plt\n    \n    def plot_distributions(original, transformed, title):\n        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))\n        \n        # Original distribution\n        original.hist(ax=ax1)\n        ax1.set_title(f'Original {title} Distribution')\n        \n        # Transformed distribution\n        transformed.hist(ax=ax2)\n        ax2.set_title(f'Transformed {title} Distribution')\n        \n        plt.tight_layout()\n        plt.show()\n    \n    # Uncomment and modify as needed for specific feature visualizations\n    # plot_distributions(final_dataframe['Annual Income'], \n    #                   transformed_data['Annual_Income_Transformed'], \n    #                   'Annual Income')\n    # plot_distributions(final_dataframe['Credit Score'], \n    #                   transformed_data['Credit_Score_Scaled'], \n                      # 'Annual Income')\n    # plot_distributions(final_dataframe['Annual Income'], \n    #                   transformed_data['Annual_Income_Transformed'], \n    #                   'Annual Income')\n    # plot_distributions(final_dataframe['Annual Income'], \n    #                   transformed_data['Annual_Income_Transformed'], \n    #                   'Annual Income')\n# Main execution guard\nif __name__ == \"__main__\":\n    main()\n\nprint(\"Feature transformation script is ready to use!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from catboost import CatBoostRegressor\nfrom sklearn.model_selection import train_test_split, RandomizedSearchCV\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport pandas as pd\nfrom scipy.stats import uniform, randint\n\n# Prepare data\nX = final_dataframe.drop('Premium Amount', axis=1)\ny = final_dataframe['Premium Amount']\n\n# Identify categorical columns\ncat_features = X.select_dtypes(include=['object', 'category']).columns.tolist()\n\n# Split data\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Initialize CatBoost Regressor\nmodel = CatBoostRegressor(loss_function='RMSE', verbose=0)\n\n# Define the parameter distribution for RandomizedSearchCV\nparam_dist = {\n    'iterations': randint(100, 2000),  # Random integer between 100 and 2000\n    'learning_rate': uniform(0.01, 0.2),  # Random float between 0.01 and 0.21\n    'depth': randint(4, 10),  # Random integer between 4 and 10\n    'l2_leaf_reg': uniform(1, 10),  # Random float between 1 and 10\n    'bagging_temperature': uniform(0, 1)  # Random float between 0 and 1\n}\n\n# Set up RandomizedSearchCV\nrandom_search = RandomizedSearchCV(estimator=model, param_distributions=param_dist, \n                                   n_iter=100, scoring='neg_mean_squared_error', \n                                   cv=3, verbose=1, n_jobs=-1, random_state=42)\n\n# Fit the model\nrandom_search.fit(X_train, y_train, cat_features=cat_features, eval_set=(X_test, y_test))\n\n# Get the best parameters\nbest_params = random_search.best_params_\nprint(f\"Best Parameters: {best_params}\")\n\n# Use the best model to make predictions\nbest_model = random_search.best_estimator_\ny_pred = best_model.predict(X_test)\n\n# Evaluate the model\nmse = mean_squared_error(y_test, y_pred)\nr2 = r2_score(y_test, y_pred)\n\nprint(f\"Mean Squared Error: {mse}\")\nprint(f\"R² Score: {r2}\")\n\n# Feature importance\nfeature_importance = best_model.get_feature_importance()\nfeature_names = X.columns\nfor name, importance in sorted(zip(feature_names, feature_importance), key=lambda x: x[1], reverse=True):\n    print(f\"{name}: {importance}\")\n\n\n# Best Parameters: {'iterations': 750, 'learning_rate': 0.1, 'depth': 8, 'l2_leaf_reg': 1, 'bagging_temperature': 1.0}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import train_test_split\nfrom scipy.stats import uniform, randint\n\ndef rmsle(y_true, y_pred):\n    y_true = np.maximum(y_true, 0)\n    y_pred = np.maximum(y_pred, 0)\n    log_true = np.log1p(y_true)\n    log_pred = np.log1p(y_pred)\n    msle = np.mean((log_true - log_pred)**2)\n    return np.sqrt(msle)\n\ndef tune_catboost_model(final_dataframe):\n    # Prepare data\n    X = final_dataframe.drop('Premium Amount', axis=1)\n    y = final_dataframe['Premium Amount']\n    \n    # Identify categorical columns\n    cat_features = X.select_dtypes(include=['object', 'category']).columns.tolist()\n    \n    # Split data\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n    \n    # Best parameters from previous search\n    best_params = {\n        'iterations': 1200,\n        'learning_rate': 0.1,\n        'depth': 8,\n        'l2_leaf_reg': 2,\n        'bagging_temperature': 0.7,\n        'random_strength': 1.0\n    }\n    \n    # Initialize and train CatBoost Regressor\n    model = CatBoostRegressor(\n        **best_params,\n        loss_function='RMSE',\n        verbose=0\n    )\n    \n    model.fit(\n        X_train, y_train, \n        cat_features=cat_features, \n        eval_set=(X_test, y_test)\n    )\n    \n    # Make predictions\n    y_pred = model.predict(X_test)\n    \n    # Calculate RMSLE\n    rmsle_score = rmsle(y_test, y_pred)\n    \n    # Get feature importance\n    feature_importance = model.get_feature_importance()\n    feature_names = X.columns\n    feature_imp_dict = dict(sorted(zip(feature_names, feature_importance), key=lambda x: x[1], reverse=True))\n    \n    # Prepare results\n    results = {\n        'params': best_params,\n        'rmsle': rmsle_score,\n        'feature_importance': feature_imp_dict\n    }\n    \n    return results\n\ndef main():\n    # Run parameter tuning\n    tuning_results = tune_catboost_model(final_dataframe)\n    \n    print(\"Model Parameters:\")\n    for param, value in tuning_results['params'].items():\n        print(f\"{param}: {value}\")\n    \n    print(f\"\\nRMSLE Score: {tuning_results['rmsle']}\")\n    \n    print(\"\\nFeature Importance:\")\n    for feature, importance in tuning_results['feature_importance'].items():\n        print(f\"{feature}: {importance}\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\n\ndef rmsle(y_true, y_pred):\n    \"\"\"\n    Calculate Root Mean Squared Logarithmic Error\n    \n    Parameters:\n    y_true (array-like): True target values\n    y_pred (array-like): Predicted target values\n    \n    Returns:\n    float: RMSLE score\n    \"\"\"\n    # Ensure non-negative values\n    y_true = np.maximum(y_true, 0)\n    y_pred = np.maximum(y_pred, 0)\n    \n    # Calculate log(1+x) to handle zero and negative values\n    log_true = np.log1p(y_true)\n    log_pred = np.log1p(y_pred)\n    \n    # Calculate mean squared logarithmic error\n    msle = np.mean((log_true - log_pred)**2)\n    \n    # Return root of mean squared logarithmic error\n    return np.sqrt(msle)\n\n# Prepare data\nX = final_dataframe.drop('Premium Amount', axis=1)\ny = final_dataframe['Premium Amount']\n\n# Identify categorical columns\ncat_features = X.select_dtypes(include=['object', 'category']).columns.tolist()\n\n# Split data\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Initialize CatBoost Regressor with best parameters\n# Use RMSE as the loss function since RMSLE is not directly supported\nmodel = CatBoostRegressor(\n    iterations=1200,\n    learning_rate=0.1,\n    depth=8,\n    l2_leaf_reg=2,\n    # bagging_temperature=1.0,\n    bagging_temperature=0.7,\n    random_strength=1.0,\n    loss_function='RMSE',  # Use RMSE instead of RMSLE\n    verbose=0\n)\n\n# Fit the model\nmodel.fit(X_train, y_train, cat_features=cat_features, eval_set=(X_test, y_test))\n\n# Make predictions\ny_pred = model.predict(X_test)\n\n# Evaluate the model using RMSLE\nrmsle_score = rmsle(y_test, y_pred)\nprint(f\"Root Mean Squared Logarithmic Error (RMSLE): {rmsle_score}\")\n\n# Feature importance\nfeature_importance = model.get_feature_importance()\nfeature_names = X.columns\nprint(\"\\nFeature Importances:\")\nfor name, importance in sorted(zip(feature_names, feature_importance), key=lambda x: x[1], reverse=True):\n    print(f\"{name}: {importance}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import lightgbm as lgb\n# from sklearn.model_selection import train_test_split, RandomizedSearchCV\n# from sklearn.preprocessing import StandardScaler\n# from scipy.stats import uniform, randint\n\n# # Direct preprocessing\n# X = final_dataframe.drop('Premium Amount', axis=1)\n# y = final_dataframe['Premium Amount']\n\n# # Split data\n# X_train, X_test, y_train, y_test = train_test_split(\n#     X, y, test_size=0.2, random_state=42\n# )\n\n# # Scale features\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # Define the model\n# model = lgb.LGBMRegressor()\n\n# # Define the parameter distribution for RandomizedSearchCV\n# param_dist = {\n#     'num_leaves': randint(20, 150),  # Random integer between 20 and 150\n#     'learning_rate': uniform(0.01, 0.1),  # Random float between 0.01 and 0.1\n#     'n_estimators': randint(100, 2000),  # Random integer between 100 and 2000\n#     'bagging_fraction': uniform(0.6, 0.4),  # Random float between 0.6 and 1.0\n#     'feature_fraction': uniform(0.6, 0.4),  # Random float between 0.6 and 1.0\n#     'bagging_freq': randint(1, 10)  # Random integer between 1 and 10\n# }\n\n# # Set up RandomizedSearchCV\n# random_search = RandomizedSearchCV(estimator=model, param_distributions=param_dist, \n#                                    n_iter=100, scoring='neg_mean_squared_error', \n#                                    cv=3, verbose=1, n_jobs=-1, random_state=42)\n\n# # Fit the model\n# random_search.fit(X_train_scaled, y_train)\n\n# # Get the best parameters\n# best_params = random_search.best_params_\n# print(f\"Best Parameters: {best_params}\")\n\n# # Use the best model to make predictions\n# best_model = random_search.best_estimator_\n# y_pred = best_model.predict(X_test_scaled)\n\n# # Evaluate the model\n# from sklearn.metrics import mean_squared_error, r2_score\n\n# mse = mean_squared_error(y_test, y_pred)\n# r2 = r2_score(y_test, y_pred)\n\n# print(f\"Mean Squared Error: {mse}\")\n# print(f\"R² Score: {r2}\")\n\n# # Feature Importance\n# import pandas as pd\n\n# importance_df = pd.DataFrame({\n#     'feature': X.columns,\n#     'importance': best_model.feature_importance()\n# }).sort_values('importance', ascending=False)\n\n# print(\"\\nTop Features:\")\n# print(importance_df.head(10))\n\n# # Additional Evaluation Metrics\n# from sklearn.metrics import mean_absolute_error, mean_absolute_percentage_error\n\n# mae = mean_absolute_error(y_test, y_pred)\n# mape = mean_absolute_percentage_error(y_test, y_pred)\n\n# print(f\"\\nMean Absolute Error: {mae}\")\n# print(f\"Mean Absolute Percentage Error: {mape:.2%}\")\n\n# # Scatter plot of predicted vs actual values\n# import matplotlib.pyplot as plt\n\n# plt.figure(figsize=(10, 6))\n# plt.scatter(y_test, y_pred, alpha=0.5)\n# plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)\n# plt.xlabel('Actual Premium Amount')\n# plt.ylabel('Predicted Premium Amount')\n# plt.title('Actual vs Predicted Premium Amount')\n# plt.tight_layout()\n# plt.show()","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# DF test\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Define encoding mappings globally\nENCODING_MAPPINGS = {\n    'Gender': {\n        'Female': 0, \n        'Male': 1\n    },\n    'Marital Status': {\n        'Married': 2, \n        'Divorced': 1, \n        'Single': 0,\n        np.nan: np.nan\n    },\n    'Education Level': {\n        'High School': 0, \n        \"Bachelor's\": 1, \n        \"Master's\": 2, \n        'PhD': 3\n    },\n    'Location': {\n        'Rural': 0, \n        'Suburban': 1, \n        'Urban': 2\n    },\n    'Smoking Status': {\n        'No': 0, \n        'Yes': 1\n    },\n    'Exercise Frequency': {\n        'Rarely': 0.5, \n        'Monthly': 1, \n        'Weekly': 4, \n        'Daily': 28\n    },\n    'Property Type': {\n        'Condo': 0, \n        'Apartment': 1, \n        'House': 2\n    },\n    'Policy Type': {\n        'Basic': 0, \n        'Comprehensive': 1, \n        'Premium': 2\n    },\n    'Occupation': {\n        np.nan: np.nan,\n        'Unemployed': 2, \n        'Employed': 4, \n        'Self-Employed': 6\n    },\n    'Customer Feedback': {\n        'Poor': 0, \n        'Average': 5, \n        'Good': 10,\n        'No Feedback': np.nan,\n        np.nan: np.nan\n    }\n}\n\ndef ordinal_encoding(df):\n    # Deep copy to avoid modifying original DataFrame\n    encoded_df = df.copy()\n    \n    # Apply Ordinal Encoding\n    for column, mapping in ENCODING_MAPPINGS.items():\n        if column in df.columns:\n            encoded_df[column] = df[column].map(mapping)\n    \n    return encoded_df\n\ndef validate_encoding(df, encoded_df):\n    \"\"\"\n    Validate the encoding process\n    \n    Parameters:\n    - df: Original DataFrame\n    - encoded_df: Encoded DataFrame\n    \n    Returns:\n    - Detailed encoding validation report\n    \"\"\"\n    print(\"\\n--- Encoding Validation Report ---\")\n    \n    for column in encoded_df.columns:\n        if column in ENCODING_MAPPINGS:\n            print(f\"\\n{column} Encoding:\")\n            \n            # Original Unique Values\n            print(\"Original Unique Values:\")\n            print(df[column].unique())\n            \n            # Encoded Unique Values\n            print(\"\\nEncoded Unique Values:\")\n            print(encoded_df[column].unique())\n            \n            # Mapping Verification\n            mapping_verification = df[column].value_counts().to_frame()\n            mapping_verification['Encoded Value'] = mapping_verification.index.map(\n                ENCODING_MAPPINGS.get(column, {})\n            )\n            \n            print(\"\\nMapping Verification:\")\n            print(mapping_verification)\n            print(\"-\" * 50)\n    \n    return encoded_df\n\ndef main(df):\n    \"\"\"\n    Main function to perform ordinal encoding\n    \n    Parameters:\n    - df: Input DataFrame\n    \n    Returns:\n    - Encoded DataFrame with validation report\n    \"\"\"\n    # Perform Ordinal Encoding\n    encoded_df = ordinal_encoding(df)\n    \n    # Validate Encoding\n    validated_df = validate_encoding(df, encoded_df)\n    \n    return validated_df\n\n# Usage\nencoded_df_test = main(df_test)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filling missing values in the encoded_df_test dataset\nencoded_df_test['Credit Score'].fillna(encoded_df_test['Credit Score'].mean(), inplace=True)\nencoded_df_test['Health Score'].fillna(encoded_df_test['Health Score'].mean(), inplace=True)\nencoded_df_test['Annual Income'].fillna(encoded_df_test['Annual Income'].median(), inplace=True)\nencoded_df_test['Number of Dependents'].fillna(encoded_df_test['Number of Dependents'].mean(), inplace=True)\nencoded_df_test['Age'].fillna(encoded_df_test['Age'].mean(), inplace=True)\nencoded_df_test['Vehicle Age'].fillna(encoded_df_test['Age'].mean(), inplace=True)\nencoded_df_test['Insurance Duration'].fillna(encoded_df_test['Insurance Duration'].mean(), inplace=True)\nencoded_df_test['Previous Claims'].fillna(encoded_df_test['Previous Claims'].median(), inplace=True)\nencoded_df_test['Marital Status'].fillna(encoded_df_test['Marital Status'].mode()[0], inplace=True)\nencoded_df_test['Occupation'].fillna(encoded_df_test['Occupation'].mode()[0], inplace=True)\nencoded_df_test['Customer Feedback'].fillna(encoded_df_test['Customer Feedback'].mode()[0], inplace=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndef create_date_features(df_date_data):\n    \"\"\"\n    Comprehensive date feature extraction with robust handling\n    \n    Parameters:\n    - df_date_data: DataFrame or Series with date information\n    \n    Returns:\n    - DataFrame with extracted date features\n    \"\"\"\n    # Ensure we're working with a DataFrame or Series\n    if not isinstance(df_date_data, (pd.DataFrame, pd.Series)):\n        raise ValueError(\"Input must be a pandas DataFrame or Series\")\n    \n    # Convert to DataFrame if it's a Series\n    if isinstance(df_date_data, pd.Series):\n        df_date_data = df_date_data.to_frame(name='Policy_Start_Date')\n    \n    # Ensure date column exists\n    if 'Policy_Start_Date' not in df_date_data.columns:\n        df_date_data = df_date_data.rename(columns={df_date_data.columns[0]: 'Policy_Start_Date'})\n    \n    # Convert to datetime with error handling\n    try:\n        df_date_data['Policy_Start_Date'] = pd.to_datetime(\n            df_date_data['Policy_Start_Date'], \n            errors='coerce'  # Convert invalid dates to NaT\n        )\n    except Exception as e:\n        print(f\"Error converting to datetime: {e}\")\n        return df_date_data\n    \n    # Remove rows with invalid dates if needed\n    df_date_data = df_date_data.dropna(subset=['Policy_Start_Date'])\n    \n    # Date Extraction Features\n    date_features = {\n        'Policy_Start_Year': df_date_data['Policy_Start_Date'].dt.year,\n        'Policy_Start_Month': df_date_data['Policy_Start_Date'].dt.month,\n        'Policy_Start_Day': df_date_data['Policy_Start_Date'].dt.day,\n        'Policy_Start_Quarter': df_date_data['Policy_Start_Date'].dt.quarter,\n        'Policy_Start_Day_of_Week': df_date_data['Policy_Start_Date'].dt.dayofweek,\n        'Policy_Start_Is_Weekend': df_date_data['Policy_Start_Date'].dt.dayofweek.isin([5,6]).astype(int)\n    }\n    \n    # Add extracted features to DataFrame\n    for feature_name, feature_data in date_features.items():\n        df_date_data[feature_name] = feature_data\n    \n    # Policy Age Calculation\n    reference_date = pd.Timestamp.now()\n    df_date_data['Policy_Age_Days'] = (reference_date - df_date_data['Policy_Start_Date']).dt.days\n    df_date_data['Policy_Age_Years'] = df_date_data['Policy_Age_Days'] / 365.25\n    \n    # Seasonality Features\n    season_map = {\n        1: 'Winter', 2: 'Winter', 3: 'Spring', \n        4: 'Spring', 5: 'Spring', 6: 'Summer', \n        7: 'Summer', 8: 'Summer', 9: 'Autumn', \n        10: 'Autumn', 11: 'Autumn', 12: 'Winter'\n    }\n    df_date_data['Policy_Start_Season'] = df_date_data['Policy_Start_Month'].map(season_map)\n    \n    # Advanced Date Features\n    df_date_data['Is_Leap_Year'] = df_date_data['Policy_Start_Date'].dt.is_leap_year\n    df_date_data['Days_In_Month'] = df_date_data['Policy_Start_Date'].dt.days_in_month\n    \n    # Date Range Percentile\n    df_date_data['Date_Range_Percentile'] = (\n        (df_date_data['Policy_Start_Date'] - df_date_data['Policy_Start_Date'].min()) / \n        (df_date_data['Policy_Start_Date'].max() - df_date_data['Policy_Start_Date'].min())\n    ) * 100\n    \n    # Comprehensive Date Analysis\n    print(\"\\n--- Comprehensive Date Feature Analysis ---\")\n    print(\"\\nDate Range:\")\n    print(f\"Earliest Date: {df_date_data['Policy_Start_Date'].min()}\")\n    print(f\"Latest Date: {df_date_data['Policy_Start_Date'].max()}\")\n    \n    print(\"\\nFeature Distribution:\")\n    for feature in ['Policy_Start_Year', 'Policy_Start_Month', 'Policy_Start_Quarter', 'Policy_Start_Season']:\n        print(f\"\\n{feature} Distribution:\")\n        print(df_date_data[feature].value_counts())\n    \n    # Optional Visualization\n    try:\n        import matplotlib.pyplot as plt\n        \n        plt.figure(figsize=(15, 10))\n        \n        # Yearly Distribution\n        plt.subplot(2, 2, 1)\n        df_date_data['Policy_Start_Year'].value_counts().sort_index().plot(kind='bar')\n        plt.title('Yearly Distribution')\n        plt.xlabel('Year')\n        plt.ylabel('Count')\n        \n        # Monthly Distribution\n        plt.subplot(2, 2, 2)\n        df_date_data['Policy_Start_Month'].value_counts().sort_index().plot(kind='bar')\n        plt.title('Monthly Distribution')\n        plt.xlabel('Month')\n        plt.ylabel('Count')\n        \n        # Quarter Distribution\n        plt.subplot(2, 2, 3)\n        df_date_data['Policy_Start_Quarter'].value_counts().sort_index().plot(kind='bar')\n        plt.title('Quarterly Distribution')\n        plt.xlabel('Quarter')\n        plt.ylabel('Count')\n        \n        # Season Distribution\n        plt.subplot(2, 2, 4)\n        df_date_data['Policy_Start_Season'].value_counts().plot(kind='pie', autopct='%1.1f%%')\n        plt.title('Seasonal Distribution')\n        \n        plt.tight_layout()\n        plt.show()\n    except Exception as viz_error:\n        print(f\"Visualization error: {viz_error}\")\n    \n    return df_date_data\n\n# Usage Examples\n# Method 1: With DataFrame\n# df_with_date_features = create_date_features(df)\n\n# Method 2: With Series\ndf_test_with_date_features = create_date_features(df_test['Policy Start Date'])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sample 20 rows\nsampled_df_test = df_test_with_date_features.sample(20)\n\n# Drop specified columns\ncolumns_to_drop = [\n    'Policy_Start_Date', \n    'Policy_Start_Quarter', \n    'Policy_Start_Day_of_Week', \n    'Policy_Start_Is_Weekend', \n    'Policy_Start_Season', \n    'Is_Leap_Year', \n    'Days_In_Month', \n    'Date_Range_Percentile',\n    'Policy_Start_Year'\n]\n\ndf_test_cleaned = df_test_with_date_features.drop(columns=columns_to_drop)\n\n\ndf_test_cleaned.sample(20)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Simple Concatenation\nfinal_test_dataframe = pd.concat([encoded_df_test, df_test_cleaned], axis=1)\n\n# Remove duplicate columns\nfinal_test_dataframe = final_test_dataframe.loc[:, ~final_test_dataframe.columns.duplicated()]\nfinal_test_dataframe.drop('Policy Start Date',axis =1, inplace = True)\n# Quick verification\nprint(final_test_dataframe.info())\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_test_dataframe.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_test_dataframe.columns","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_dataframe.columns","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_dataframe.sample(10)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions = model.predict(final_test_dataframe)\n\n# Create submission dataframe\n# Assuming df_test has an 'id' column\nsubmission = pd.DataFrame({\n    'id': final_test_dataframe['id'],  # Make sure to use the correct id column\n    'Premium Amount': predictions\n})\n\n# Save submission to CSV\nsubmission.to_csv('predictions_catboost_transformed.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}