{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":101849,"databundleVersionId":13093295,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"!pip install pqdm\nimport pandas as pd\nimport numpy as np\nimport pickle\nimport itertools\nfrom scipy.signal import savgol_filter\nfrom astropy.stats import sigma_clip\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\nimport gc  # add garbage collector","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T09:21:12.341315Z","iopub.execute_input":"2025-08-14T09:21:12.341569Z","iopub.status.idle":"2025-08-14T09:21:25.50481Z","shell.execute_reply.started":"2025-08-14T09:21:12.341547Z","shell.execute_reply":"2025-08-14T09:21:25.503897Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configuration Class","metadata":{"execution":{"iopub.status.busy":"2025-08-13T20:53:30.057292Z","iopub.execute_input":"2025-08-13T20:53:30.058105Z","iopub.status.idle":"2025-08-13T20:53:30.063534Z","shell.execute_reply.started":"2025-08-13T20:53:30.058061Z","shell.execute_reply":"2025-08-13T20:53:30.06223Z"}}},{"cell_type":"code","source":"class Config:\n    DATA_PATH = '/kaggle/input/ariel-data-challenge-2025'\n    \n    DATASET = \"train\" \n\n    CUT_INF = 39\n    CUT_SUP = 250\n    SENSOR_CONFIG = {\n        \"AIRS-CH0\": {\n            \"raw_shape\": [11250, 32, 356],\n            \"calibrated_shape\": [1, 32, 250 - 39],\n            \"linear_corr_shape\": (6, 32, 356),\n            \"dt_pattern\": (0.1, 4.5), \n            \"binning\": 30\n        },\n        \"FGS1\": {\n            \"raw_shape\": [135000, 32, 32],\n            \"calibrated_shape\": [1, 32, 32],\n            \"linear_corr_shape\": (6, 32, 32),\n            \"dt_pattern\": (0.1, 0.1),\n            \"binning\": 30 * 12\n        }\n    }\n    MODEL_PHASE_DETECTION_SLICE = slice(30, 140)\n    \n    N_JOBS = 4","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T09:21:25.505873Z","iopub.execute_input":"2025-08-14T09:21:25.506511Z","iopub.status.idle":"2025-08-14T09:21:25.513191Z","shell.execute_reply.started":"2025-08-14T09:21:25.506478Z","shell.execute_reply":"2025-08-14T09:21:25.511858Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Classes for data analysis","metadata":{}},{"cell_type":"code","source":"class SignalProcessor:\n    def __init__(self, config):\n        self.cfg = config\n        self.adc_info = pd.read_csv(f\"{self.cfg.DATA_PATH}/adc_info.csv\")\n        self.planet_ids = pd.read_csv(f'{self.cfg.DATA_PATH}/{self.cfg.DATASET}_star_info.csv', index_col='planet_id').index.astype(int)\n\n    def _apply_linear_corr(self, linear_corr, signal):\n        linear_corr_flipped = np.flip(linear_corr, axis=0)\n        corrected_signal = signal.copy()\n        \n        for x, y in itertools.product(range(signal.shape[1]), range(signal.shape[2])):\n            poly = np.poly1d(linear_corr_flipped[:, x, y])\n            corrected_signal[:, x, y] = poly(corrected_signal[:, x, y])\n            \n        return corrected_signal\n    \n    def _calibrate_single_signal(self, planet_id, sensor):\n        sensor_cfg = self.cfg.SENSOR_CONFIG[sensor]\n        data_root = f\"{self.cfg.DATA_PATH}/{self.cfg.DATASET}/{planet_id}\"\n        \n        signal = pd.read_parquet(f\"{data_root}/{sensor}_signal_0.parquet\").to_numpy()\n        dark = pd.read_parquet(f\"{data_root}/{sensor}_calibration_0/dark.parquet\").to_numpy()\n        dead = pd.read_parquet(f\"{data_root}/{sensor}_calibration_0/dead.parquet\").to_numpy()\n        flat = pd.read_parquet(f\"{data_root}/{sensor}_calibration_0/flat.parquet\").to_numpy()\n        linear_corr = pd.read_parquet(f\"{data_root}/{sensor}_calibration_0/linear_corr.parquet\").values.astype(np.float64).reshape(sensor_cfg[\"linear_corr_shape\"])\n\n        signal = signal.reshape(sensor_cfg[\"raw_shape\"])\n        gain = self.adc_info[f\"{sensor}_adc_gain\"].iloc[0]\n        offset = self.adc_info[f\"{sensor}_adc_offset\"].iloc[0]\n        signal = signal / gain + offset\n\n        hot = sigma_clip(dark, sigma=5, maxiters=5).mask\n\n        if sensor == \"AIRS-CH0\":\n            signal = signal[:, :, self.cfg.CUT_INF : self.cfg.CUT_SUP]\n            linear_corr = linear_corr[:, :, self.cfg.CUT_INF : self.cfg.CUT_SUP]\n            dark = dark[:, self.cfg.CUT_INF : self.cfg.CUT_SUP]\n            dead = dead[:, self.cfg.CUT_INF : self.cfg.CUT_SUP]\n            flat = flat[:, self.cfg.CUT_INF : self.cfg.CUT_SUP]\n            hot = hot[:, self.cfg.CUT_INF : self.cfg.CUT_SUP]\n        \n        base_dt, increment = sensor_cfg[\"dt_pattern\"]\n        dt = np.ones(len(signal)) * base_dt\n        dt[1::2] += increment\n        \n        signal = signal.clip(0)\n        signal = self._apply_linear_corr(linear_corr, signal)\n        signal -= dark * dt[:, np.newaxis, np.newaxis]\n        \n        flat = flat.reshape(sensor_cfg[\"calibrated_shape\"])\n        flat[dead.reshape(sensor_cfg[\"calibrated_shape\"])] = np.nan\n        flat[hot.reshape(sensor_cfg[\"calibrated_shape\"])] = np.nan\n        \n        signal = signal / flat\n        return signal\n\n    def _preprocess_calibrated_signal(self, calibrated_signal, sensor):\n        sensor_cfg = self.cfg.SENSOR_CONFIG[sensor]\n        binning = sensor_cfg[\"binning\"]\n\n        if sensor == \"AIRS-CH0\":\n            signal_roi = calibrated_signal[:, 10:22, :]\n        else: # FGS1\n            signal_roi = calibrated_signal[:, 10:22, 10:22]\n            signal_roi = signal_roi.reshape(signal_roi.shape[0], -1)\n        \n        mean_signal = np.nanmean(signal_roi, axis=1)\n        cds_signal = mean_signal[1::2] - mean_signal[0::2]\n\n        n_bins = cds_signal.shape[0] // binning\n        binned = np.array([cds_signal[j*binning : (j+1)*binning].mean(axis=0) for j in range(n_bins)])\n        if sensor == \"FGS1\":\n            binned = binned.reshape((binned.shape[0], 1))\n        return binned\n\n    def _process_planet_sensor(self, args):\n        planet_id, sensor = args['planet_id'], args['sensor']\n        calibrated = self._calibrate_single_signal(planet_id, sensor)\n        preprocessed = self._preprocess_calibrated_signal(calibrated, sensor)\n        return preprocessed\n\n    def process_all_data(self):\n        all_preprocessed_signals = []\n        \n        # tqdm - это просто красивый индикатор прогресса\n        for planet_id in tqdm(self.planet_ids, desc=\"Обработка сигналов планет\"):\n            fgs_signal = self._process_planet_sensor({'planet_id': planet_id, 'sensor': 'FGS1'})\n            airs_signal = self._process_planet_sensor({'planet_id': planet_id, 'sensor': 'AIRS-CH0'})\n            \n            combined_signal = np.concatenate([fgs_signal, airs_signal], axis=1)\n            all_preprocessed_signals.append(combined_signal)\n            \n            gc.collect()\n            \n        return np.stack(all_preprocessed_signals, axis=0).astype(np.float32)\n\nclass FeatureExtractor:\n    def __init__(self, config):\n        self.cfg = config\n        self.signal_processor = SignalProcessor(config)\n\n    def _phase_detector(self, signal):\n        smooth_signal = savgol_filter(signal, 20, 2)\n        search_slice = self.cfg.MODEL_PHASE_DETECTION_SLICE\n        min_idx = np.argmin(smooth_signal[search_slice]) + search_slice.start\n        \n        grad1 = np.gradient(smooth_signal[:min_idx])\n        grad2 = np.gradient(smooth_signal[min_idx:])\n        \n        phase1 = np.argmin(grad1) if len(grad1) > 0 else 0\n        phase2 = np.argmax(grad2) + min_idx if len(grad2) > 0 else len(signal) - 1\n        return phase1, phase2\n\n    def create_features_for_planet(self, planet_id, preprocessed_signal, star_info):\n        features_list = []\n        fgs_signal = preprocessed_signal[:, 0]\n        phase1, phase2 = self._phase_detector(fgs_signal)\n        \n        planet_star_info = star_info.loc[planet_id].to_dict()\n        transit_features = {'phase1': phase1, 'phase2': phase2, 'transit_duration': phase2 - phase1}\n        \n        for wave_idx in range(preprocessed_signal.shape[1]):\n            wave_signal = preprocessed_signal[:, wave_idx]\n            features = {'planet_id': planet_id, 'wave_index': wave_idx}\n            features.update(planet_star_info)\n            features.update(transit_features)\n            \n            features['signal_mean'] = np.nanmean(wave_signal)\n            features['signal_std'] = np.nanstd(wave_signal)\n            \n            in_transit_signal = wave_signal[phase1:phase2]\n            out_of_transit_signal = np.concatenate([wave_signal[:phase1], wave_signal[phase2:]])\n            features['in_transit_mean'] = np.nanmean(in_transit_signal) if len(in_transit_signal) > 0 else np.nan\n            features['out_of_transit_mean'] = np.nanmean(out_of_transit_signal) if len(out_of_transit_signal) > 0 else np.nan\n            features['depth_crude'] = features['out_of_transit_mean'] - features['in_transit_mean']\n            \n            features_list.append(features)\n        return pd.DataFrame(features_list)\n\n    def process_all_and_create_feature_table(self):\n        print(\"Начинаем предобработку сигналов (последовательно)...\")\n        self.signal_processor = SignalProcessor(Config())\n        preprocessed_data = self.signal_processor.process_all_data() # Уже оптимизированный\n        \n        star_info = pd.read_csv(f'{self.cfg.DATA_PATH}/{self.cfg.DATASET}_star_info.csv', index_col='planet_id')\n        \n        # --- ИЗМЕНЕНИЕ: Обрабатываем чанками для еще большей экономии ---\n        print(\"Создаем таблицу с признаками...\")\n        all_features_list = []\n        for i, pid in tqdm(enumerate(star_info.index), total=len(star_info), desc=\"Feature Engineering\"):\n            # Получаем фичи для одной планеты\n            planet_features_df = self.create_features_for_planet(pid, preprocessed_data[i], star_info)\n            all_features_list.append(planet_features_df)\n\n            # Периодически очищаем память\n            if i % 50 == 0:\n                gc.collect()\n\n        # Собираем все в один DataFrame в конце\n        feature_table = pd.concat(all_features_list, ignore_index=True)\n\n        # Оптимизируем типы данных в итоговой таблице\n        for col in feature_table.select_dtypes(include=['float64']).columns:\n            feature_table[col] = feature_table[col].astype(np.float32)\n        for col in feature_table.select_dtypes(include=['int64']).columns:\n            feature_table[col] = feature_table[col].astype(np.int32)\n        \n        return feature_table","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T09:24:10.194661Z","iopub.execute_input":"2025-08-14T09:24:10.194984Z","iopub.status.idle":"2025-08-14T09:24:10.224188Z","shell.execute_reply.started":"2025-08-14T09:24:10.194961Z","shell.execute_reply":"2025-08-14T09:24:10.222995Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ML pipeline","metadata":{}},{"cell_type":"code","source":"config = Config()\n\nfeature_extractor = FeatureExtractor(config)\nX_train = feature_extractor.process_all_and_create_feature_table()\n\ntargets = pd.read_csv(f'{config.DATA_PATH}/train.csv', index_col='planet_id')\ny_train_flat = targets.drop(columns=['sigma']).values.flatten().astype(np.float32) # Оптимизация\nX_train['s'] = y_train_flat\nX_train = X_train.dropna(subset=['s'])\n\ngc.collect()\n\nTARGET_COL = 's'\nfeatures_cols = [col for col in X_train.columns if col not in ['planet_id', TARGET_COL]]\n\nlgb_params = {\n    'objective': 'regression_l1', 'metric': 'mae', 'n_estimators': 2000,\n    'learning_rate': 0.02, 'feature_fraction': 0.8, 'bagging_fraction': 0.8,\n    'bagging_freq': 1, 'lambda_l1': 0.1, 'lambda_l2': 0.1,\n    'num_leaves': 31, 'verbose': -1, 'n_jobs': -1, 'seed': 42}\nmodel = lgb.LGBMRegressor(**lgb_params)\nmodel.fit(X_train[features_cols], X_train[TARGET_COL])\n\nMODEL_OUTPUT_PATH = \"lgbm_model.pkl\"\nwith open(MODEL_OUTPUT_PATH, 'wb') as f:\n    pickle.dump(model, f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T09:24:13.945762Z","iopub.execute_input":"2025-08-14T09:24:13.946254Z","iopub.status.idle":"2025-08-14T09:27:49.687773Z","shell.execute_reply.started":"2025-08-14T09:24:13.946212Z","shell.execute_reply":"2025-08-14T09:27:49.685993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}