{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":87191,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":73238,"modelId":98108}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div style=\"text-align:center; margin: 10px 0;\">\n    <h2>✨ If you found this notebook insightful, please like! ✨</h2>\n\n</div>\n","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA","metadata":{}},{"cell_type":"code","source":"train_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_adc_info.csv')\ntest_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/test_adc_info.csv')\ntrain_labels = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_labels.csv')\nwavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')\n\ntrain_adc_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:39.928231Z","iopub.execute_input":"2024-08-02T03:57:39.928639Z","iopub.status.idle":"2024-08-02T03:57:40.084932Z","shell.execute_reply.started":"2024-08-02T03:57:39.92861Z","shell.execute_reply":"2024-08-02T03:57:40.084016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_adc_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:40.086887Z","iopub.execute_input":"2024-08-02T03:57:40.087193Z","iopub.status.idle":"2024-08-02T03:57:40.097329Z","shell.execute_reply.started":"2024-08-02T03:57:40.087167Z","shell.execute_reply":"2024-08-02T03:57:40.096328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:40.098503Z","iopub.execute_input":"2024-08-02T03:57:40.099316Z","iopub.status.idle":"2024-08-02T03:57:40.129843Z","shell.execute_reply.started":"2024-08-02T03:57:40.099284Z","shell.execute_reply":"2024-08-02T03:57:40.12882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wavelengths.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:40.130948Z","iopub.execute_input":"2024-08-02T03:57:40.131212Z","iopub.status.idle":"2024-08-02T03:57:40.154215Z","shell.execute_reply.started":"2024-08-02T03:57:40.13119Z","shell.execute_reply":"2024-08-02T03:57:40.15292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"axis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')\naxis_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:40.156602Z","iopub.execute_input":"2024-08-02T03:57:40.156871Z","iopub.status.idle":"2024-08-02T03:57:40.195389Z","shell.execute_reply.started":"2024-08-02T03:57:40.156848Z","shell.execute_reply":"2024-08-02T03:57:40.194505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_adc_info.describe()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:40.196385Z","iopub.execute_input":"2024-08-02T03:57:40.196966Z","iopub.status.idle":"2024-08-02T03:57:40.22438Z","shell.execute_reply.started":"2024-08-02T03:57:40.196939Z","shell.execute_reply":"2024-08-02T03:57:40.223561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_adc_info.info()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:40.22543Z","iopub.execute_input":"2024-08-02T03:57:40.225721Z","iopub.status.idle":"2024-08-02T03:57:40.24289Z","shell.execute_reply.started":"2024-08-02T03:57:40.225698Z","shell.execute_reply":"2024-08-02T03:57:40.241881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_adc_info.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:40.244138Z","iopub.execute_input":"2024-08-02T03:57:40.24455Z","iopub.status.idle":"2024-08-02T03:57:40.254971Z","shell.execute_reply.started":"2024-08-02T03:57:40.244494Z","shell.execute_reply":"2024-08-02T03:57:40.254022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(style=\"whitegrid\")\nfig, ax = plt.subplots(1, 2, figsize=(14, 7))\nsns.histplot(train_adc_info['FGS1_adc_gain'], bins=30, color='dodgerblue', kde=True, ax=ax[0])\nax[0].set_title('Distribution of Gain', fontsize=14)\nax[0].set_xlabel('Gain', fontsize=12)\nax[0].set_ylabel('Frequency', fontsize=12)\n\nsns.histplot(train_adc_info['FGS1_adc_offset'], bins=30, color='seagreen', kde=True, ax=ax[1])\nax[1].set_title('Distribution of Offset', fontsize=14)\nax[1].set_xlabel('Offset', fontsize=12)\nax[1].set_ylabel('Frequency', fontsize=12)\n\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:40.256022Z","iopub.execute_input":"2024-08-02T03:57:40.256268Z","iopub.status.idle":"2024-08-02T03:57:41.779081Z","shell.execute_reply.started":"2024-08-02T03:57:40.256247Z","shell.execute_reply":"2024-08-02T03:57:41.778157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def restore_dynamic_range(signal, gain, offset):\n    return signal * gain + offset","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:41.780108Z","iopub.execute_input":"2024-08-02T03:57:41.780381Z","iopub.status.idle":"2024-08-02T03:57:41.784591Z","shell.execute_reply.started":"2024-08-02T03:57:41.780357Z","shell.execute_reply":"2024-08-02T03:57:41.783697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signal_data = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/train/2633183716/AIRS-CH0_signal.parquet')\nadc_info = train_adc_info[train_adc_info['planet_id'] == 2633183716]","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:41.785762Z","iopub.execute_input":"2024-08-02T03:57:41.786019Z","iopub.status.idle":"2024-08-02T03:57:42.804178Z","shell.execute_reply.started":"2024-08-02T03:57:41.785997Z","shell.execute_reply":"2024-08-02T03:57:42.803211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"restored_signal = restore_dynamic_range(signal_data, adc_info['FGS1_adc_gain'].values[0], adc_info['FGS1_adc_offset'].values[0])","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:42.805586Z","iopub.execute_input":"2024-08-02T03:57:42.806197Z","iopub.status.idle":"2024-08-02T03:57:43.262835Z","shell.execute_reply.started":"2024-08-02T03:57:42.806164Z","shell.execute_reply":"2024-08-02T03:57:43.261733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(restored_signal.iloc[0].values.reshape(32, 356), cmap='viridis')\nplt.colorbar()\nplt.title('Restored Signal Frame')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:43.264425Z","iopub.execute_input":"2024-08-02T03:57:43.26482Z","iopub.status.idle":"2024-08-02T03:57:43.720757Z","shell.execute_reply.started":"2024-08-02T03:57:43.264788Z","shell.execute_reply":"2024-08-02T03:57:43.719798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modelling","metadata":{}},{"cell_type":"code","source":"class ExoplanetDataset(Dataset):\n    def __init__(self, signal_data, labels):\n        self.signal_data = signal_data\n        self.labels = labels\n\n    def __len__(self):\n        return len(self.signal_data)\n\n    def __getitem__(self, idx):\n        x = self.signal_data[idx]\n        y = self.labels[idx]\n        return x, y\n","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:43.827881Z","iopub.execute_input":"2024-08-02T03:57:43.828134Z","iopub.status.idle":"2024-08-02T03:57:43.840569Z","shell.execute_reply.started":"2024-08-02T03:57:43.828113Z","shell.execute_reply":"2024-08-02T03:57:43.839664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_data(planet_id):\n    signal_data = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/train/{planet_id}/AIRS-CH0_signal.parquet')\n    adc_info = train_adc_info[train_adc_info['planet_id'] == planet_id]\n    restored_signal = restore_dynamic_range(signal_data, adc_info['FGS1_adc_gain'].values[0], adc_info['FGS1_adc_offset'].values[0])\n    return restored_signal\n","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:43.841763Z","iopub.execute_input":"2024-08-02T03:57:43.842137Z","iopub.status.idle":"2024-08-02T03:57:43.850012Z","shell.execute_reply.started":"2024-08-02T03:57:43.842108Z","shell.execute_reply":"2024-08-02T03:57:43.849066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"planet_id = 2633183716\nrestored_signal = prepare_data(planet_id)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:43.850968Z","iopub.execute_input":"2024-08-02T03:57:43.851207Z","iopub.status.idle":"2024-08-02T03:57:45.522373Z","shell.execute_reply.started":"2024-08-02T03:57:43.851186Z","shell.execute_reply":"2024-08-02T03:57:45.521252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Length of restored_signal: {len(restored_signal)}\")\nprint(f\"Length of train_labels: {len(train_labels)}\")","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:45.523727Z","iopub.execute_input":"2024-08-02T03:57:45.524055Z","iopub.status.idle":"2024-08-02T03:57:45.531298Z","shell.execute_reply.started":"2024-08-02T03:57:45.524026Z","shell.execute_reply":"2024-08-02T03:57:45.53018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signal_length = len(restored_signal)\nlabel_length = train_labels.shape[0]\n","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:45.532637Z","iopub.execute_input":"2024-08-02T03:57:45.53299Z","iopub.status.idle":"2024-08-02T03:57:45.541895Z","shell.execute_reply.started":"2024-08-02T03:57:45.532964Z","shell.execute_reply":"2024-08-02T03:57:45.54038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if signal_length > label_length:\n    restored_signal = restored_signal[:label_length]\nelif label_length > signal_length:\n    train_labels = train_labels.iloc[:signal_length]\n\nprint(f\"Adjusted length of restored_signal: {len(restored_signal)}\")\nprint(f\"Adjusted length of train_labels: {len(train_labels)}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:45.543474Z","iopub.execute_input":"2024-08-02T03:57:45.544301Z","iopub.status.idle":"2024-08-02T03:57:45.552296Z","shell.execute_reply.started":"2024-08-02T03:57:45.544268Z","shell.execute_reply":"2024-08-02T03:57:45.551276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = ExoplanetDataset(restored_signal.values.reshape(-1, 1, 32, 356), train_labels.iloc[:, 1:].values)\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:45.553245Z","iopub.execute_input":"2024-08-02T03:57:45.553475Z","iopub.status.idle":"2024-08-02T03:57:45.56605Z","shell.execute_reply.started":"2024-08-02T03:57:45.553453Z","shell.execute_reply":"2024-08-02T03:57:45.564117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### I will build more complex model in future updates","metadata":{}},{"cell_type":"code","source":"class ExoplanetModel(nn.Module):\n    def __init__(self):\n        super(ExoplanetModel, self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, kernel_size=3, padding=1)\n        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)\n        \n        # Calculate the size after convolutions\n        self.conv_output_size = 64 * 32 * 356\n        \n        self.fc1 = nn.Linear(self.conv_output_size, 128)\n        self.fc2 = nn.Linear(128, 283)  # 283 wavelengths\n\n    def forward(self, x):\n        x = torch.relu(self.conv1(x))\n        x = torch.relu(self.conv2(x))\n        x = x.view(x.size(0), -1)  # Flatten the tensor\n        x = torch.relu(self.fc1(x))\n        x = self.fc2(x)\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:45.567399Z","iopub.execute_input":"2024-08-02T03:57:45.567937Z","iopub.status.idle":"2024-08-02T03:57:45.576222Z","shell.execute_reply.started":"2024-08-02T03:57:45.567912Z","shell.execute_reply":"2024-08-02T03:57:45.575438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ExoplanetModel()\ncriterion = nn.MSELoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:45.57735Z","iopub.execute_input":"2024-08-02T03:57:45.578141Z","iopub.status.idle":"2024-08-02T03:57:46.590667Z","shell.execute_reply.started":"2024-08-02T03:57:45.578115Z","shell.execute_reply":"2024-08-02T03:57:46.589571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# num_epochs = 10\n# for epoch in range(num_epochs):\n#     model.train()\n#     total_loss = 0\n#     for x, y in train_loader:\n#         x, y = x.float(), y.float()\n#         optimizer.zero_grad()\n#         outputs = model(x)\n#         loss = criterion(outputs, y)\n#         loss.backward()\n#         optimizer.step()\n#         total_loss += loss.item()\n#     print(f'Epoch {epoch+1}/{num_epochs}, Loss: {total_loss/len(train_loader)}')\n\n# torch.save(model.state_dict(), 'exoplanet_model.pth')","metadata":{"execution":{"iopub.status.busy":"2024-08-02T03:57:46.592067Z","iopub.execute_input":"2024-08-02T03:57:46.592403Z","iopub.status.idle":"2024-08-02T04:09:35.378295Z","shell.execute_reply.started":"2024-08-02T03:57:46.592377Z","shell.execute_reply":"2024-08-02T04:09:35.377182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ExoplanetModel()\nmodel_path = '/kaggle/input/exoplanet-model/pytorch/default/1/exoplanet_model.pth'\nmodel.load_state_dict(torch.load(model_path))\nmodel.eval()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T04:49:54.658968Z","iopub.execute_input":"2024-08-02T04:49:54.659347Z","iopub.status.idle":"2024-08-02T04:49:58.823936Z","shell.execute_reply.started":"2024-08-02T04:49:54.659318Z","shell.execute_reply":"2024-08-02T04:49:58.823053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_test_data(planet_id):\n    signal_data = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/test/{planet_id}/AIRS-CH0_signal.parquet')\n    adc_info = test_adc_info[test_adc_info['planet_id'] == planet_id]  # Assuming test_adc_info is available\n    restored_signal = restore_dynamic_range(signal_data, adc_info['FGS1_adc_gain'].values[0], adc_info['FGS1_adc_offset'].values[0])\n    return restored_signal\n\nplanet_id_test = 499191466 \nrestored_signal_test = prepare_test_data(planet_id_test)\nrestored_signal_test","metadata":{"execution":{"iopub.status.busy":"2024-08-02T04:27:02.915066Z","iopub.execute_input":"2024-08-02T04:27:02.915944Z","iopub.status.idle":"2024-08-02T04:27:04.355475Z","shell.execute_reply.started":"2024-08-02T04:27:02.915901Z","shell.execute_reply":"2024-08-02T04:27:04.35423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"planet_id_test = 499191466  \nrestored_signal_test = prepare_test_data(planet_id_test)\n\nrestored_signal_test = restored_signal_test.values.reshape(-1, 1, 32, 356)\n\ntest_dataset = ExoplanetDataset(restored_signal_test, np.zeros((restored_signal_test.shape[0], 283)))  # Dummy labels\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-08-02T04:27:13.951669Z","iopub.execute_input":"2024-08-02T04:27:13.952035Z","iopub.status.idle":"2024-08-02T04:27:15.435594Z","shell.execute_reply.started":"2024-08-02T04:27:13.952004Z","shell.execute_reply":"2024-08-02T04:27:15.434499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate_model(model, data_loader):\n    model.eval()\n    total_gll = 0\n    with torch.no_grad():\n        for x, _ in data_loader: \n            x = x.float()\n            outputs = model(x).numpy()\n            y_true = np.zeros_like(outputs)  \n            sigma_user = np.ones_like(outputs) * 1e-5 \n            gll = -0.5 * (np.log(2 * np.pi) + np.log(sigma_user**2) + ((y_true - outputs)**2 / sigma_user**2))\n            total_gll += gll.sum()\n    return total_gll","metadata":{"execution":{"iopub.status.busy":"2024-08-02T04:27:21.567423Z","iopub.execute_input":"2024-08-02T04:27:21.568502Z","iopub.status.idle":"2024-08-02T04:27:21.574781Z","shell.execute_reply.started":"2024-08-02T04:27:21.568464Z","shell.execute_reply":"2024-08-02T04:27:21.573808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gll_value = evaluate_model(model, test_loader)\nprint(f'Gaussian Log-Likelihood: {gll_value}')","metadata":{"execution":{"iopub.status.busy":"2024-08-02T04:34:18.739945Z","iopub.execute_input":"2024-08-02T04:34:18.741002Z","iopub.status.idle":"2024-08-02T04:36:19.06731Z","shell.execute_reply.started":"2024-08-02T04:34:18.740965Z","shell.execute_reply":"2024-08-02T04:36:19.06648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_spectra(y_true, y_pred, wavelengths):\n    plt.figure(figsize=(10, 6))\n    if y_true is not None:\n        plt.plot(wavelengths, y_true, label='True Spectrum', linestyle='--', color='blue')\n    plt.plot(wavelengths, y_pred, label='Predicted Spectrum', linestyle='-', color='red')\n    plt.xlabel('Wavelength')\n    plt.ylabel('Intensity')\n    plt.title('Exoplanet Atmospheric Spectrum')\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-02T04:38:03.566571Z","iopub.execute_input":"2024-08-02T04:38:03.566937Z","iopub.status.idle":"2024-08-02T04:38:03.574223Z","shell.execute_reply.started":"2024-08-02T04:38:03.566908Z","shell.execute_reply":"2024-08-02T04:38:03.572849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x, _ in test_loader:\n    x = x.float()\n    outputs = model(x)\n    y_pred = outputs.detach().numpy()\n    wavelengths = np.arange(283)\n    plot_spectra(None, y_pred[0], wavelengths)\n    break","metadata":{"execution":{"iopub.status.busy":"2024-08-02T04:38:34.581929Z","iopub.execute_input":"2024-08-02T04:38:34.582361Z","iopub.status.idle":"2024-08-02T04:38:35.438682Z","shell.execute_reply.started":"2024-08-02T04:38:34.582328Z","shell.execute_reply":"2024-08-02T04:38:35.437443Z"},"trusted":true},"execution_count":null,"outputs":[]}]}