{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":101849,"databundleVersionId":12846694,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split, cross_val_score, KFold\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.preprocessing import StandardScaler\nimport xgboost as xgb\nimport catboost as cb\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-07T18:09:21.656803Z","iopub.execute_input":"2025-07-07T18:09:21.657106Z","iopub.status.idle":"2025-07-07T18:09:21.661757Z","shell.execute_reply.started":"2025-07-07T18:09:21.657082Z","shell.execute_reply":"2025-07-07T18:09:21.660838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nclass ArielEnsembleModel:\n    def __init__(self, fast_mode=False):\n        \"\"\"\n        Initialize the ensemble model with XGBoost and CatBoost\n        fast_mode: If True, use faster parameters for Kaggle submission\n        \"\"\"\n        self.fast_mode = fast_mode  # <--- Added this line\n\n        # XGBoost parameters - TUNABLE PARAMETERS MARKED WITH COMMENTS\n        if self.fast_mode: # Changed to self.fast_mode\n            # Faster parameters for Kaggle submission\n            self.xgb_params = {\n                'objective': 'reg:squarederror',\n                'n_estimators': 300,   # Reduced for faster training\n                'max_depth': 6,        # Reduced for faster training\n                'learning_rate': 0.1,  # Increased for faster convergence\n                'subsample': 0.8,      # TUNABLE: Try 0.6-0.9\n                'colsample_bytree': 0.8, # TUNABLE: Try 0.6-0.9\n                'reg_alpha': 0.1,      # TUNABLE: Try 0.0-1.0 (L1 regularization)\n                'reg_lambda': 0.1,     # TUNABLE: Try 0.0-1.0 (L2 regularization)\n                'min_child_weight': 1, # TUNABLE: Try 1-10\n                'gamma': 0,            # TUNABLE: Try 0-0.5\n                'random_state': 42,\n                'n_jobs': -1,\n                'tree_method': 'hist', # Faster tree method\n                'verbose': 0\n            }\n        else:\n            # Full parameters for better accuracy\n            self.xgb_params = {\n                'objective': 'reg:squarederror',\n                'n_estimators': 1000,  # TUNABLE: Try 500-3000\n                'max_depth': 8,        # TUNABLE: Try 4-12\n                'learning_rate': 0.05, # TUNABLE: Try 0.01-0.3\n                'subsample': 0.8,      # TUNABLE: Try 0.6-0.9\n                'colsample_bytree': 0.8, # TUNABLE: Try 0.6-0.9\n                'reg_alpha': 0.1,      # TUNABLE: Try 0.0-1.0 (L1 regularization)\n                'reg_lambda': 0.1,     # TUNABLE: Try 0.0-1.0 (L2 regularization)\n                'min_child_weight': 1, # TUNABLE: Try 1-10\n                'gamma': 0,            # TUNABLE: Try 0-0.5\n                'random_state': 42,\n                'n_jobs': -1,\n                'tree_method': 'hist',\n                'early_stopping_rounds': 50,\n                'verbose': 0\n            }\n\n        # CatBoost parameters - TUNABLE PARAMETERS MARKED WITH COMMENTS\n        if self.fast_mode: # Changed to self.fast_mode\n            # Faster parameters for Kaggle submission\n            self.cb_params = {\n                'iterations': 300,     # Reduced for faster training\n                'depth': 6,            # Reduced for faster training\n                'learning_rate': 0.1,  # Increased for faster convergence\n                'l2_leaf_reg': 3,      # TUNABLE: Try 1-10\n                'random_strength': 1,  # TUNABLE: Try 0-10\n                'bootstrap_type': 'Bernoulli', # TUNABLE: Try 'Bernoulli', 'MVS', 'Poisson'\n                'subsample': 0.8,      # TUNABLE: Try 0.6-0.9\n                'random_state': 42,\n                'verbose': 0,\n                'allow_writing_files': False,  # Important for Kaggle\n                'thread_count': -1\n            }\n        else:\n            # Full parameters for better accuracy\n            self.cb_params = {\n                'iterations': 1000,     # TUNABLE: Try 500-3000\n                'depth': 8,            # TUNABLE: Try 4-12\n                'learning_rate': 0.05, # TUNABLE: Try 0.01-0.3\n                'l2_leaf_reg': 3,      # TUNABLE: Try 1-10\n                'random_strength': 1,   # TUNABLE: Try 0-10\n                'bootstrap_type': 'Bernoulli', # TUNABLE: Try 'Bernoulli', 'MVS', 'Poisson'\n                'subsample': 0.8,       # TUNABLE: Try 0.6-0.9\n                'random_state': 42,\n                'verbose': 0,\n                'early_stopping_rounds': 50,\n                'allow_writing_files': False,  # Important for Kaggle\n                'thread_count': -1\n            }\n\n        # Alternative CatBoost configurations for different bootstrap types\n        # Uncomment one of these if you want to try different bootstrap methods:\n\n        # Option 1: Bayesian bootstrap (no subsample, has bagging_temperature)\n        # self.cb_params = {\n        #     'iterations': 1000,\n        #     'depth': 8,\n        #     'learning_rate': 0.05,\n        #     'l2_leaf_reg': 3,\n        #     'bagging_temperature': 1,  # Only for Bayesian\n        #     'random_strength': 1,\n        #     'bootstrap_type': 'Bayesian',\n        #     'random_state': 42,\n        #     'verbose': 0,\n        #     'early_stopping_rounds': 50\n        # }\n\n        # Option 2: MVS bootstrap (has subsample, no bagging_temperature)\n        # self.cb_params = {\n        #     'iterations': 1000,\n        #     'depth': 8,\n        #     'learning_rate': 0.05,\n        #     'l2_leaf_reg': 3,\n        #     'random_strength': 1,\n        #     'bootstrap_type': 'MVS',\n        #     'subsample': 0.8,\n        #     'random_state': 42,\n        #     'verbose': 0,\n        #     'early_stopping_rounds': 50\n        # }\n        self.xgb_weight = 0.5  # TUNABLE: Try 0.3-0.7\n        self.cb_weight = 0.5   # TUNABLE: Try 0.3-0.7\n\n        self.models = {}\n        self.scalers = {}\n        self.target_cols = []\n\n    def load_data(self, data_path='/kaggle/input/ariel-data-challenge-2025/'):\n        \"\"\"\n        Load and prepare the dataset\n        \"\"\"\n        print(\"Loading data...\")\n\n        # Load main datasets\n        self.train_df = pd.read_csv(f'{data_path}train.csv')\n        self.test_df = pd.read_csv(f'{data_path}test_star_info.csv')\n\n        # Load auxiliary data\n        self.train_star_info = pd.read_csv(f'{data_path}train_star_info.csv')\n        self.wavelengths = pd.read_csv(f'{data_path}wavelengths.csv')\n        self.adc_info = pd.read_csv(f'{data_path}adc_info.csv')\n\n        print(f\"Train shape: {self.train_df.shape}\")\n        print(f\"Test shape: {self.test_df.shape}\")\n        print(f\"Train star info shape: {self.train_star_info.shape}\")\n        print(f\"Wavelengths shape: {self.wavelengths.shape}\")\n\n        # Get target columns (all columns except planet_id in train.csv)\n        self.target_cols = [col for col in self.train_df.columns if col != 'planet_id']\n        print(f\"Number of target columns: {len(self.target_cols)}\")\n\n    def feature_engineering(self):\n        \"\"\"\n        Create additional features from star information and wavelengths\n        \"\"\"\n        print(\"Performing feature engineering...\")\n\n        # Merge star information with train and test data\n        self.train_features = self.train_star_info.copy()\n        self.test_features = self.test_df.copy()\n\n        # Create additional features from star parameters\n        for df in [self.train_features, self.test_features]:\n            # Stellar ratios and combinations - TUNABLE: Add more domain-specific features\n            df['Rs_Ms_ratio'] = df['Rs'] / (df['Ms'] + 1e-8)\n            df['Ts_Mp_ratio'] = df['Ts'] / (df['Mp'] + 1e-8)\n            df['e_P_product'] = df['e'] * df['P']\n            df['sma_P_ratio'] = df['sma'] / (df['P'] + 1e-8)\n            df['stellar_luminosity'] = df['Rs']**2 * (df['Ts']/5778)**4  # Relative to Sun\n            df['planet_insolation'] = df['stellar_luminosity'] / (df['sma']**2 + 1e-8)\n            df['equilibrium_temp'] = df['Ts'] * np.sqrt(df['Rs']/(2*df['sma'] + 1e-8))\n\n            # Polynomial features for important parameters - TUNABLE: Adjust degree\n            df['Ts_squared'] = df['Ts']**2\n            df['Rs_squared'] = df['Rs']**2\n            df['Mp_squared'] = df['Mp']**2\n            df['log_P'] = np.log(df['P'] + 1e-8)\n            df['log_sma'] = np.log(df['sma'] + 1e-8)\n\n        # Statistical features from wavelength data - TUNABLE: Add more statistical features\n        wavelength_stats = self.wavelengths.describe().T\n        for stat in ['mean', 'std', 'min', 'max']:\n            self.train_features[f'wavelength_{stat}'] = wavelength_stats[stat].iloc[0]\n            self.test_features[f'wavelength_{stat}'] = wavelength_stats[stat].iloc[0]\n\n        # Remove planet_id for modeling\n        feature_cols = [col for col in self.train_features.columns if col != 'planet_id']\n        self.X_train = self.train_features[feature_cols]\n        self.X_test = self.test_features[feature_cols]\n\n        print(f\"Feature engineering complete. Total features: {len(feature_cols)}\")\n\n    def train_models(self):\n        \"\"\"\n        Train XGBoost and CatBoost models for each target\n        \"\"\"\n        print(\"Training models...\")\n\n        # Prepare targets\n        y_train = self.train_df[self.target_cols]\n\n        # Use smaller validation split for fast mode\n        test_size = 0.1 if self.fast_mode else 0.2 # Changed to self.fast_mode\n\n        # Split for validation\n        X_train, X_val, y_train_split, y_val_split = train_test_split(\n            self.X_train, y_train, test_size=test_size, random_state=42\n        )\n\n        # Scale features - TUNABLE: Try different scalers or no scaling\n        self.scaler = StandardScaler()\n        X_train_scaled = self.scaler.fit_transform(X_train)\n        X_val_scaled = self.scaler.transform(X_val)\n        X_test_scaled = self.scaler.transform(self.X_test)\n\n        # Train models for each target\n        self.xgb_models = {}\n        self.cb_models = {}\n\n        # Use early stopping for non-fast mode\n        use_early_stopping = not self.fast_mode # Changed to self.fast_mode\n\n        for i, target_col in enumerate(self.target_cols):\n            if i % 50 == 0:\n                print(f\"Training models for target {i+1}/{len(self.target_cols)}: {target_col}\")\n\n            # Train XGBoost\n            xgb_model = xgb.XGBRegressor(**self.xgb_params)\n            if use_early_stopping:\n                xgb_model.fit(\n                    X_train_scaled, y_train_split[target_col],\n                    eval_set=[(X_val_scaled, y_val_split[target_col])],\n                    verbose=False\n                )\n            else:\n                xgb_model.fit(X_train_scaled, y_train_split[target_col])\n            self.xgb_models[target_col] = xgb_model\n\n            # Train CatBoost\n            cb_model = cb.CatBoostRegressor(**self.cb_params)\n            if use_early_stopping:\n                cb_model.fit(\n                    X_train_scaled, y_train_split[target_col],\n                    eval_set=(X_val_scaled, y_val_split[target_col]),\n                    verbose=False\n                )\n            else:\n                cb_model.fit(X_train_scaled, y_train_split[target_col])\n            self.cb_models[target_col] = cb_model\n\n        # Store scaled test features\n        self.X_test_scaled = X_test_scaled\n\n        print(\"Model training complete!\")\n\n    def predict(self):\n        \"\"\"\n        Make predictions using ensemble of XGBoost and CatBoost\n        \"\"\"\n        print(\"Making predictions...\")\n\n        predictions = {}\n\n        for target_col in self.target_cols:\n            # Get predictions from both models\n            xgb_pred = self.xgb_models[target_col].predict(self.X_test_scaled)\n            cb_pred = self.cb_models[target_col].predict(self.X_test_scaled)\n\n            # Ensemble predictions - TUNABLE: Try different ensemble methods\n            ensemble_pred = (self.xgb_weight * xgb_pred + self.cb_weight * cb_pred)\n            predictions[target_col] = ensemble_pred\n\n        return predictions\n\n    def evaluate_models(self):\n        \"\"\"\n        Evaluate model performance using cross-validation\n        \"\"\"\n        print(\"Evaluating models...\")\n\n        # Sample a few targets for evaluation (to save time)\n        sample_targets = self.target_cols[:10]  # Evaluate first 10 targets\n\n        cv_scores = {}\n        kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n\n        # Create parameter sets for cross-validation, explicitly removing early_stopping_rounds\n        # since cross_val_score doesn't provide an eval_set for it.\n        xgb_cv_params = self.xgb_params.copy()\n        if 'early_stopping_rounds' in xgb_cv_params:\n            del xgb_cv_params['early_stopping_rounds']\n        if 'verbose' in xgb_cv_params and xgb_cv_params['verbose'] == 0: # Ensure verbose is not 0 for potential warnings if needed, but for CV typically kept silent\n             del xgb_cv_params['verbose'] # Remove verbose for clean CV output if 0\n\n        cb_cv_params = self.cb_params.copy()\n        if 'early_stopping_rounds' in cb_cv_params:\n            del cb_cv_params['early_stopping_rounds']\n        if 'verbose' in cb_cv_params and cb_cv_params['verbose'] == 0: # Same for CatBoost\n            del cb_cv_params['verbose'] # Remove verbose if 0 for clean CV output\n\n\n        for target_col in sample_targets:\n            y_target = self.train_df[target_col]\n\n            # XGBoost CV\n            xgb_model = xgb.XGBRegressor(**xgb_cv_params) # Use the modified params\n            xgb_scores = cross_val_score(\n                xgb_model, self.X_train, y_target,\n                cv=kfold, scoring='neg_mean_squared_error'\n            )\n\n            # CatBoost CV\n            cb_model = cb.CatBoostRegressor(**cb_cv_params) # Use the modified params\n            cb_scores = cross_val_score(\n                cb_model, self.X_train, y_target,\n                cv=kfold, scoring='neg_mean_squared_error'\n            )\n\n            cv_scores[target_col] = {\n                'xgb_rmse': np.sqrt(-xgb_scores.mean()),\n                'cb_rmse': np.sqrt(-cb_scores.mean()),\n                'xgb_std': np.sqrt(xgb_scores.std()),\n                'cb_std': np.sqrt(cb_scores.std())\n            }\n\n        # Print evaluation results\n        print(\"\\nCross-validation Results (Sample):\")\n        print(\"=\" * 60)\n        for target, scores in cv_scores.items():\n            print(f\"{target[:20]:20} | XGB: {scores['xgb_rmse']:.4f}±{scores['xgb_std']:.4f} | \"\n                  f\"CB: {scores['cb_rmse']:.4f}±{scores['cb_std']:.4f}\")\n\n        return cv_scores\n\n    def create_submission(self, predictions, filename='submission.csv'):\n        \"\"\"\n        Create submission file in the required format\n        \"\"\"\n        print(f\"Creating submission file: {filename}\")\n\n        # Create submission dataframe\n        submission_df = pd.DataFrame()\n        submission_df['planet_id'] = self.test_features['planet_id']\n\n        # Add predictions for all targets\n        for target_col in self.target_cols:\n            submission_df[target_col] = predictions[target_col]\n\n        # Save submission\n        submission_df.to_csv(filename, index=False)\n        print(f\"Submission file saved with shape: {submission_df.shape}\")\n\n        return submission_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-07T18:48:00.505324Z","iopub.execute_input":"2025-07-07T18:48:00.506068Z","iopub.status.idle":"2025-07-07T18:48:00.532523Z","shell.execute_reply.started":"2025-07-07T18:48:00.506031Z","shell.execute_reply":"2025-07-07T18:48:00.531861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def main():\n    \"\"\"\n    Main execution function\n    \"\"\"\n    import os\n    \n    # Use fast mode for Kaggle submission to avoid timeout\n    fast_mode = os.environ.get('KAGGLE_KERNEL_RUN_TYPE') == 'Interactive'\n    \n    # Initialize model\n    model = ArielEnsembleModel(fast_mode=fast_mode)\n    \n    if fast_mode:\n        print(\"Running in FAST MODE for Kaggle submission\")\n    else:\n        print(\"Running in FULL MODE for better accuracy\")\n    \n    # Load data\n    model.load_data()\n    \n    # Feature engineering\n    model.feature_engineering()\n    \n    # Evaluate models (optional - skip in fast mode)\n    if not fast_mode:\n        model.evaluate_models()\n    \n    # Train models\n    model.train_models()\n    \n    # Make predictions\n    predictions = model.predict()\n    \n    # Create submission\n    submission = model.create_submission(predictions)\n    \n    print(\"Pipeline completed successfully!\")\n    \n    # Display feature importance for first target (XGBoost)\n    if len(model.target_cols) > 0:\n        first_target = model.target_cols[0]\n        importance = model.xgb_models[first_target].feature_importances_\n        feature_names = [col for col in model.X_train.columns]\n        \n        print(f\"\\nTop 10 Feature Importances for {first_target}:\")\n        print(\"=\" * 50)\n        importance_df = pd.DataFrame({\n            'feature': feature_names,\n            'importance': importance\n        }).sort_values('importance', ascending=False)\n        \n        for i, row in importance_df.head(10).iterrows():\n            print(f\"{row['feature']:30} | {row['importance']:.4f}\")\ndef run_kaggle_submission():\n    \"\"\"\n    Optimized function for Kaggle submission\n    \"\"\"\n    # Initialize model in fast mode\n    model = ArielEnsembleModel(fast_mode=True)\n    \n    # Load data (adjust path if needed)\n    model.load_data('/kaggle/input/ariel-data-challenge-2025/')\n    \n    # Feature engineering\n    model.feature_engineering()\n    \n    # Train models\n    model.train_models()\n    \n    # Make predictions\n    predictions = model.predict()\n    \n    # Create submission\n    submission = model.create_submission(predictions, '/kaggle/working/submission.csv')\n    \n    print(\"Kaggle submission ready!\")\n    return submission\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-07T18:48:04.752198Z","iopub.execute_input":"2025-07-07T18:48:04.75249Z","iopub.status.idle":"2025-07-07T18:53:16.426139Z","shell.execute_reply.started":"2025-07-07T18:48:04.752467Z","shell.execute_reply":"2025-07-07T18:53:16.425291Z"}},"outputs":[],"execution_count":null}]}