{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":101849,"databundleVersionId":13093295,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n \nNeurIPS - reconstructing the spectra of exoplanets \n
\n\n### NeurIPS - solution for recovering exoplanet spectra with uncertainty estimates and physical models.
\n\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\n\n# Пути к файлам (укажите свои пути)\ntrain_csv_path = '/kaggle/input/ariel-data-challenge-2025/train.csv'\nstar_info_path = '/kaggle/input/ariel-data-challenge-2025/test_star_info.csv'\nsignal_path_template = '/kaggle/input/ariel-data-challenge-2025/train/1010375142/FGS1_signal_0.parquet'\ncalibration_path_template = '/kaggle/input/ariel-data-challenge-2025/train/1010375142/AIRS-CH0_calibration_0/flat.parquet'\n\n# 1. Загрузка данных с обработкой ошибок\ntry:\n print(\"Загружаю train_df...\")\n train_df = pd.read_csv(train_csv_path)\n print(f\"train_df загружен. Количество строк: {len(train_df)}\")\nexcept Exception as e:\n print(\"Ошибка при загрузке train_df:\", e)\n\ntry:\n print(\"Загружаю star_info_df...\")\n star_info_df = pd.read_csv(star_info_path)\n print(f\"star_info_df загружен. Количество строк: {len(star_info_df)}\")\nexcept Exception as e:\n print(\"Ошибка при загрузке star_info_df:\", e)\n\n# 2. Функция восстановления сигнала\ndef restore_signal(parquet_file, gain, offset):\n try:\n data_uint16 = pd.read_parquet(parquet_file).values\n data_float = data_uint16.astype(np.float64) / gain + offset\n return data_float\n except Exception as e:\n print(f\"Ошибка при восстановлении сигнала из {parquet_file}: {e}\")\n return np.zeros(1000) # или подходящий размер по умолчанию\n\n# 3. Создание Dataset\nclass ExoDataset(Dataset):\n def __init__(self, df, star_info_df, data_type='train'):\n self.df = df\n self.star_info_df = star_info_df\n self.data_type = data_type\n\n def __len__(self):\n return len(self.df)\n\n def __getitem__(self, idx):\n row = self.df.iloc[idx]\n planet_id = row['planet_id']\n\n # Получение параметров звезды\n matches = self.star_info_df[self.star_info_df['planet_id'] == planet_id]\n if len(matches) > 0:\n star_params_row = matches.iloc[0]\n star_params = torch.tensor(star_params_row.values, dtype=torch.float)\n else:\n star_params = torch.zeros(5, dtype=torch.float)\n\n obs_idx = 0\n # Возможно, нужно вставить planet_id в путь\n air_signal_path = signal_path_template\n fgs_signal_path = signal_path_template\n\n # Пути к калибровкам\n dark_air_path = calibration_path_template\n dark_fgs_path = calibration_path_template\n\n # Загрузка темных кадров с обработкой ошибок\n try:\n dark_air = pd.read_parquet(dark_air_path).values\n except Exception as e:\n print(f\"Ошибка при загрузке dark_air: {e}\")\n dark_air = np.zeros(1000)\n\n try:\n dark_fgs = pd.read_parquet(dark_fgs_path).values\n except Exception as e:\n print(f\"Ошибка при загрузке dark_fgs: {e}\")\n dark_fgs = np.zeros(1000)\n\n # Определение gain и offset (здесь можно вставить ваши реальные параметры)\n gain_air, offset_air = 1.0, 0.0\n gain_fgs, offset_fgs = 1.0, 0.0\n\n # Восстановление сигналов\n air_signal = restore_signal(air_signal_path, gain_air, offset_air)\n fgs_signal = restore_signal(fgs_signal_path, gain_fgs, offset_fgs)\n\n # Обработка dark — усреднение\n dark_air_mean = np.mean(dark_air)\n dark_fgs_mean = np.mean(dark_fgs)\n\n # Корректировка сигналов\n air_signal_corrected = air_signal - dark_air_mean\n fgs_signal_corrected = fgs_signal - dark_fgs_mean\n\n return {\n 'planet_id': planet_id,\n 'star_params': star_params,\n 'air_signal': torch.tensor(air_signal_corrected, dtype=torch.float),\n 'fgs_signal': torch.tensor(fgs_signal_corrected, dtype=torch.float)\n }\n\n# Создаем датасет и DataLoader\ndataset = ExoDataset(train_df, star_info_df)\ndataloader = DataLoader(dataset, batch_size=8, shuffle=True) # уменьшил batch_size для стабильности\n\n# Проверка загрузки первого элемента\ntry:\n sample = dataset[0]\n print(\"Первый образец успешно загружен.\")\n print(\"Размер air_signal:\", sample['air_signal'].shape)\n print(\"Размер fgs_signal:\", sample['fgs_signal'].shape)\n print(\"Параметры звезды:\", sample['star_params'].shape)\nexcept Exception as e:\n print(\"Ошибка при получении первого элемента:\", e)\n\n# 4. Определение модели\nsample_item = dataset[0]\ninput_dim = sample_item['air_signal'].numel() + sample_item['fgs_signal'].numel()\noutput_dim = len(sample_item['star_params'])\n\nclass SimpleNet(nn.Module):\n def __init__(self, input_dim, output_dim):\n super().__init__()\n self.fc = nn.Sequential(\n nn.Linear(input_dim, 512),\n nn.ReLU(),\n nn.Linear(512, output_dim)\n )\n def forward(self, x):\n return self.fc(x)\n\nmodel = SimpleNet(input_dim=input_dim, output_dim=output_dim)\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\ncriterion = nn.MSELoss()\n\n# 5. Обучение\nfor epoch in range(10):\n total_loss = 0\n for batch_idx, batch in enumerate(dataloader):\n try:\n air_signals = batch['air_signal']\n fgs_signals = batch['fgs_signal']\n star_params = batch['star_params']\n\n inputs = torch.cat([air_signals.view(air_signals.size(0), -1),\n fgs_signals.view(fgs_signals.size(0), -1)], dim=1)\n\n preds = model(inputs)\n loss = criterion(preds, star_params)\n\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n total_loss += loss.item()\n\n if batch_idx % 10 == 0:\n print(f\"Epoch {epoch+1}, Batch {batch_idx}, Loss: {loss.item():.6f}\")\n except Exception as e:\n print(f\"Ошибка во время обучения на батче {batch_idx}: {e}\")\n print(f\"Эпоха {epoch+1} завершена. СреднийLoss: {total_loss/len(dataloader):.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T17:31:34.94962Z","iopub.execute_input":"2025-08-12T17:31:34.949888Z","execution_failed":"2025-08-12T17:32:38.807Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Solving the problem of recovering an exoplanet spectrum from noisy data is a multi-task problem that involves pre-processing the data, training a model, and generating predictions. Below, I will provide an example plan and code to help you get started. It includes:\n* Loading and processing the data\n* Restoring the original signals with calibration\n* Training a basic model to predict the spectrum and uncertainty\n* Generating predictions for the test set","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nfrom tqdm import tqdm\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Пути к файлам\ntrain_csv_path = '/kaggle/input/ariel-data-challenge-2025/train.csv'\nwavelengths_path = '/kaggle/input/ariel-data-challenge-2025/wavelengths.csv'\nadc_info_path = '/kaggle/input/ariel-data-challenge-2025/adc_info.csv'\nstar_info_path = '/kaggle/input/ariel-data-challenge-2025/test_star_info.csv'\nsignal_path_template = '/kaggle/input/ariel-data-challenge-2025/train/1010375142/FGS1_signal_0.parquet'\ncalibration_path_template = '/kaggle/input/ariel-data-challenge-2025/train/1010375142/AIRS-CH0_calibration_0/linear_corr.parquet'\n\n# Загрузка метаданных\ntrain_df = pd.read_csv(train_csv_path)\nwavelengths_df = pd.read_csv(wavelengths_path)\nadc_df = pd.read_csv(adc_info_path)\nstar_info_df = pd.read_csv(star_info_path)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:28:48.222069Z","iopub.execute_input":"2025-08-12T10:28:48.222333Z","iopub.status.idle":"2025-08-12T10:28:48.338489Z","shell.execute_reply.started":"2025-08-12T10:28:48.222315Z","shell.execute_reply":"2025-08-12T10:28:48.337729Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ၊၊||၊ Restoring the original signals | \n\n\n\nThe data is saved as uint16, and it needs to be converted using the ADC parameters:","metadata":{}},{"cell_type":"code","source":"def restore_signal(parquet_file, gain, offset):\n data_uint16 = pd.read_parquet(parquet_file).values\n data_float = data_uint16.astype(np.float64) / gain + offset\n return data_float","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:28:52.385707Z","iopub.execute_input":"2025-08-12T10:28:52.38613Z","iopub.status.idle":"2025-08-12T10:28:52.397734Z","shell.execute_reply.started":"2025-08-12T10:28:52.386099Z","shell.execute_reply":"2025-08-12T10:28:52.393694Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ၊၊||၊ Creating a dataset | \n\n\n\n","metadata":{}},{"cell_type":"code","source":"class ExoDataset(Dataset):\n def __init__(self, df, data_type='train'):\n self.df = df\n self.data_type = data_type\n\n def __len__(self):\n return len(self.df)\n\n def __getitem__(self, idx):\n row = self.df.iloc[idx]\n planet_id = row['planet_id']\n star_params = star_info_df[star_info_df['planet_id'] == planet_id].iloc[0]\n \n # Загрузка сигналов\n # Предполагается, что есть список наблюдений\n # Для простоты возьмем одно наблюдение\n obs_idx = 0 # или случайный/по условию\n air_signal_path = signal_path_template.format(planet_id=planet_id, instrument='AIRS-CH0', obs=obs_idx)\n fgs_signal_path = signal_path_template.format(planet_id=planet_id, instrument='FGS1', obs=obs_idx)\n \n # Загрузка калибровки\n dark_air = pd.read_parquet(calibration_path_template.format(planet_id=planet_id, instrument='AIRS-CH0', calib_type='dark')).values\n dark_fgs = pd.read_parquet(calibration_path_template.format(planet_id=planet_id, instrument='FGS1', calib_type='dark')).values\n gain_air, offset_air = 1.0, 0.0 # взять из adc_info или из файла\n gain_fgs, offset_fgs = 1.0, 0.0\n \n # Восстановление сигналов\n air_signal = restore_signal(air_signal_path, gain_air, offset_air) - dark_air\n fgs_signal = restore_signal(fgs_signal_path, gain_fgs, offset_fgs) - dark_fgs\n \n # Возвращаем спектр и параметры\n return {\n 'planet_id': planet_id,\n 'star_params': star_params,\n 'air_signal': air_signal,\n 'fgs_signal': fgs_signal\n }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:28:56.067249Z","iopub.execute_input":"2025-08-12T10:28:56.06756Z","iopub.status.idle":"2025-08-12T10:28:56.075671Z","shell.execute_reply.started":"2025-08-12T10:28:56.06754Z","shell.execute_reply":"2025-08-12T10:28:56.074666Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ၊၊||၊ A simple model for spectrum prediction | \n\n\n","metadata":{}},{"cell_type":"code","source":"class SimpleNet(nn.Module):\n def __init__(self, input_dim, output_dim):\n super().__init__()\n self.fc = nn.Sequential(\n nn.Linear(input_dim, 512),\n nn.ReLU(),\n nn.Linear(512, output_dim)\n )\n \n def forward(self, x):\n return self.fc(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:29:02.714944Z","iopub.execute_input":"2025-08-12T10:29:02.715224Z","iopub.status.idle":"2025-08-12T10:29:02.720863Z","shell.execute_reply.started":"2025-08-12T10:29:02.715204Z","shell.execute_reply":"2025-08-12T10:29:02.719778Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ၊၊||၊ Model training | \n\n\n","metadata":{}},{"cell_type":"code","source":"dataset = ExoDataset(train_df)\ndataloader = DataLoader(dataset, batch_size=16, shuffle=True)\n\nmodel = SimpleNet(input_dim=2*32*32, output_dim=283) # пример, зависит от формы данных\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\ncriterion = nn.MSELoss()\n\nfor epoch in range(10):\n for batch in tqdm(dataloader):\n # Объединяем сигналы\n inputs = torch.cat([batch['air_signal'].view(batch_size, -1),\n batch['fgs_signal'].view(batch_size, -1)], dim=1).float()\n targets = torch.tensor(batch['star_params'].values).float() # или спектры из train.csv\n preds = model(inputs)\n loss = criterion(preds, targets)\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:29:10.491106Z","iopub.execute_input":"2025-08-12T10:29:10.491384Z","iopub.status.idle":"2025-08-12T10:29:10.537937Z","shell.execute_reply.started":"2025-08-12T10:29:10.491365Z","shell.execute_reply":"2025-08-12T10:29:10.536538Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ၊၊||၊ Summary | \n\n\n\n* To improve accuracy, you will need to:\n* Use physical spectrum models for training\n* Implement uncertainty prediction (e.g., through Dropout or Bayesian NN)\n* Process multiple observations (stacking)\n* Optimize the architecture and hyperparameters","metadata":{}},{"cell_type":"code","source":"def physical_model_spectrum(star_params, wavelength):\n # star_params: R, M, T, Mp, e, P, sma, i\n R = star_params['Rs']\n T = star_params['Ts']\n # Простая модель: черное тело с температурой T, масштабированное R\n # В реальности — более сложная химическая модель\n spectrum = (wavelength ** -1) * np.exp(- (wavelength / T) ** 2) * R\n return spectrum","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:04:37.268699Z","iopub.execute_input":"2025-08-12T10:04:37.268965Z","iopub.status.idle":"2025-08-12T10:04:37.27402Z","shell.execute_reply.started":"2025-08-12T10:04:37.268948Z","shell.execute_reply":"2025-08-12T10:04:37.273081Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ၊၊||၊ Creating a physical model of the spectrum | \n\n\n","metadata":{}},{"cell_type":"code","source":"def generate_synthetic_data(star_params, wavelength):\n spectrum = physical_model_spectrum(star_params, wavelength)\n # Добавляем шум\n noise = np.random.normal(0, 0.05, size=spectrum.shape)\n noisy_spectrum = spectrum + noise\n return noisy_spectrum","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:04:40.537193Z","iopub.execute_input":"2025-08-12T10:04:40.537494Z","iopub.status.idle":"2025-08-12T10:04:40.542379Z","shell.execute_reply.started":"2025-08-12T10:04:40.537434Z","shell.execute_reply":"2025-08-12T10:04:40.541672Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ၊၊||၊ Synthetic data generation (for training) | \n\n\n","metadata":{}},{"cell_type":"code","source":"class ExoSynthDataset(Dataset):\n def __init__(self, star_params_df, wavelengths, num_obs=3):\n self.star_params_df = star_params_df\n self.wavelengths = wavelengths\n self.num_obs = num_obs\n\n def __len__(self):\n return len(self.star_params_df)\n\n def __getitem__(self, idx):\n star_params = self.star_params_df.iloc[idx]\n spectrum_list = []\n for _ in range(self.num_obs):\n spectrum = generate_synthetic_data(star_params, self.wavelengths)\n spectrum_list.append(spectrum)\n # Склейка наблюдений\n combined_spectrum = np.concatenate(spectrum_list)\n # Целевая — параметры звезды\n target = star_params[['Rs', 'Ms', 'Ts']].values\n return {\n 'input': combined_spectrum,\n 'target': target,\n 'star_params': star_params\n }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:04:42.697037Z","iopub.execute_input":"2025-08-12T10:04:42.697303Z","iopub.status.idle":"2025-08-12T10:04:42.703688Z","shell.execute_reply.started":"2025-08-12T10:04:42.697286Z","shell.execute_reply":"2025-08-12T10:04:42.702647Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ၊၊||၊ Dataset with multiple observations | \n\n","metadata":{}},{"cell_type":"code","source":"class PhysNetDropout(nn.Module):\n def __init__(self, input_dim, output_dim, dropout_prob=0.3):\n super().__init__()\n self.fc1 = nn.Linear(input_dim, 512)\n self.dropout1 = nn.Dropout(dropout_prob)\n self.fc2 = nn.Linear(512, 256)\n self.dropout2 = nn.Dropout(dropout_prob)\n self.fc3 = nn.Linear(256, output_dim)\n\n def forward(self, x):\n x = F.relu(self.fc1(x))\n x = self.dropout1(x)\n x = F.relu(self.fc2(x))\n x = self.dropout2(x)\n return self.fc3(x)\n\n def predict_with_uncertainty(self, x, n_samples=100):\n # Предсказываем несколько раз с Dropout для оценки неопределенности\n preds = []\n self.train() # включаем Dropout\n with torch.no_grad():\n for _ in range(n_samples):\n preds.append(self.forward(x).cpu().numpy())\n preds = np.array(preds)\n mean_pred = preds.mean(axis=0)\n uncertainty = preds.std(axis=0)\n return mean_pred, uncertainty","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:04:45.63122Z","iopub.execute_input":"2025-08-12T10:04:45.631532Z","iopub.status.idle":"2025-08-12T10:04:45.639157Z","shell.execute_reply.started":"2025-08-12T10:04:45.631504Z","shell.execute_reply":"2025-08-12T10:04:45.638058Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ၊၊||၊ A model with Dropout for prediction | \n\n\n\n","metadata":{}},{"cell_type":"code","source":"# Параметры\nwavelengths = np.linspace(1.0, 3.0, 283) # пример диапазона\ndataset = ExoSynthDataset(star_params_df=star_info_df, wavelengths=wavelengths, num_obs=3)\ndataloader = DataLoader(dataset, batch_size=16, shuffle=True)\n\nmodel = PhysNetDropout(input_dim=3*283, output_dim=3) # предсказываем 3 параметра\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\nepochs = 20\n\nloss_fn = nn.MSELoss()\n\nfor epoch in range(epochs):\n total_loss = 0\n for batch in tqdm(dataloader):\n inputs = torch.tensor(batch['input']).float()\n targets = torch.tensor(batch['target']).float()\n\n preds = model(inputs)\n loss = loss_fn(preds, targets)\n\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n total_loss += loss.item()\n print(f\"Epoch {epoch+1}/{epochs}, Loss: {total_loss/len(dataloader)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:04:48.822727Z","iopub.execute_input":"2025-08-12T10:04:48.822991Z","iopub.status.idle":"2025-08-12T10:04:48.870774Z","shell.execute_reply.started":"2025-08-12T10:04:48.822972Z","shell.execute_reply":"2025-08-12T10:04:48.869201Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ၊၊||၊ Prediction and visualization | \n\n\n\nThe data is saved as uint16, and it needs to be converted using the ADC parameters:","metadata":{}},{"cell_type":"code","source":"# Берем один образец\ntest_idx = 0\ntest_sample = dataset[test_idx]\ninput_tensor = torch.tensor(test_sample['input']).unsqueeze(0).float()\n\n# Предсказание с оценкой неопределенности\nmodel.eval()\nmean_pred, uncertainty = model.predict_with_uncertainty(input_tensor, n_samples=100)\n\n# Визуализация спектра\nplt.figure(figsize=(10,6))\nplt.plot(wavelengths, test_sample['input'][:283], label='Зашумленный спектр')\nplt.plot(wavelengths, physical_model_spectrum(test_sample['star_params'], wavelengths), label='Истинный спектр', linestyle='--')\nplt.plot(wavelengths, mean_pred[0]*wavelengths, label='Предсказанный спектр')\nplt.fill_between(wavelengths, \n (mean_pred[0] - 2*uncertainty[0])*wavelengths,\n (mean_pred[0] + 2*uncertainty[0])*wavelengths,\n color='gray', alpha=0.3, label='Доверительный интервал')\nplt.xlabel('Wavelength (µm)')\nplt.ylabel('Flux')\nplt.legend()\nplt.title('Предсказание спектра с неопределенностью')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T10:04:53.899599Z","iopub.execute_input":"2025-08-12T10:04:53.899891Z","iopub.status.idle":"2025-08-12T10:04:54.237086Z","shell.execute_reply.started":"2025-08-12T10:04:53.899872Z","shell.execute_reply":"2025-08-12T10:04:54.235516Z"}},"outputs":[],"execution_count":null}]}