{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":87191,"sourceType":"modelInstanceVersion","modelInstanceId":73238,"modelId":98108}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:45:56.196385Z","iopub.execute_input":"2024-09-02T21:45:56.196669Z","iopub.status.idle":"2024-09-02T21:46:01.956779Z","shell.execute_reply.started":"2024-09-02T21:45:56.196643Z","shell.execute_reply":"2024-09-02T21:46:01.955921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA","metadata":{}},{"cell_type":"markdown","source":"Hello","metadata":{}},{"cell_type":"code","source":"train_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_adc_info.csv')\ntest_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/test_adc_info.csv')\ntrain_labels = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_labels.csv')\nwavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')\n\ntrain_adc_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:01.958461Z","iopub.execute_input":"2024-09-02T21:46:01.958873Z","iopub.status.idle":"2024-09-02T21:46:02.098233Z","shell.execute_reply.started":"2024-09-02T21:46:01.958846Z","shell.execute_reply":"2024-09-02T21:46:02.097322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_adc_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:02.102472Z","iopub.execute_input":"2024-09-02T21:46:02.102731Z","iopub.status.idle":"2024-09-02T21:46:02.112825Z","shell.execute_reply.started":"2024-09-02T21:46:02.102707Z","shell.execute_reply":"2024-09-02T21:46:02.111964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:02.116212Z","iopub.execute_input":"2024-09-02T21:46:02.116797Z","iopub.status.idle":"2024-09-02T21:46:02.145929Z","shell.execute_reply.started":"2024-09-02T21:46:02.116765Z","shell.execute_reply":"2024-09-02T21:46:02.14506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wavelengths.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:02.147261Z","iopub.execute_input":"2024-09-02T21:46:02.147528Z","iopub.status.idle":"2024-09-02T21:46:02.168913Z","shell.execute_reply.started":"2024-09-02T21:46:02.147505Z","shell.execute_reply":"2024-09-02T21:46:02.168004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"axis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')\naxis_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:02.17013Z","iopub.execute_input":"2024-09-02T21:46:02.170458Z","iopub.status.idle":"2024-09-02T21:46:02.355045Z","shell.execute_reply.started":"2024-09-02T21:46:02.170413Z","shell.execute_reply":"2024-09-02T21:46:02.354135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_adc_info.describe()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:02.356204Z","iopub.execute_input":"2024-09-02T21:46:02.356497Z","iopub.status.idle":"2024-09-02T21:46:02.382361Z","shell.execute_reply.started":"2024-09-02T21:46:02.356471Z","shell.execute_reply":"2024-09-02T21:46:02.381485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_adc_info.info()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:02.383725Z","iopub.execute_input":"2024-09-02T21:46:02.384Z","iopub.status.idle":"2024-09-02T21:46:02.399714Z","shell.execute_reply.started":"2024-09-02T21:46:02.383977Z","shell.execute_reply":"2024-09-02T21:46:02.398814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_adc_info.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:02.400867Z","iopub.execute_input":"2024-09-02T21:46:02.401209Z","iopub.status.idle":"2024-09-02T21:46:02.412508Z","shell.execute_reply.started":"2024-09-02T21:46:02.401179Z","shell.execute_reply":"2024-09-02T21:46:02.411648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(style=\"whitegrid\")\nfig, ax = plt.subplots(1, 2, figsize=(14, 7))\nsns.histplot(train_adc_info['FGS1_adc_gain'], bins=30, color='dodgerblue', kde=True, ax=ax[0])\nax[0].set_title('Distribution of Gain', fontsize=14)\nax[0].set_xlabel('Gain', fontsize=12)\nax[0].set_ylabel('Frequency', fontsize=12)\n\nsns.histplot(train_adc_info['FGS1_adc_offset'], bins=30, color='seagreen', kde=True, ax=ax[1])\nax[1].set_title('Distribution of Offset', fontsize=14)\nax[1].set_xlabel('Offset', fontsize=12)\nax[1].set_ylabel('Frequency', fontsize=12)\n\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:02.413757Z","iopub.execute_input":"2024-09-02T21:46:02.414766Z","iopub.status.idle":"2024-09-02T21:46:03.571987Z","shell.execute_reply.started":"2024-09-02T21:46:02.414731Z","shell.execute_reply":"2024-09-02T21:46:03.571082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def restore_dynamic_range(signal, gain, offset):\n    return signal * gain + offset","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:03.573456Z","iopub.execute_input":"2024-09-02T21:46:03.573849Z","iopub.status.idle":"2024-09-02T21:46:03.578475Z","shell.execute_reply.started":"2024-09-02T21:46:03.57382Z","shell.execute_reply":"2024-09-02T21:46:03.577631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signal_data = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/train/2633183716/AIRS-CH0_signal.parquet')\nadc_info = train_adc_info[train_adc_info['planet_id'] == 2633183716]","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:03.579855Z","iopub.execute_input":"2024-09-02T21:46:03.58029Z","iopub.status.idle":"2024-09-02T21:46:05.824202Z","shell.execute_reply.started":"2024-09-02T21:46:03.580258Z","shell.execute_reply":"2024-09-02T21:46:05.823215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"restored_signal = restore_dynamic_range(signal_data, adc_info['FGS1_adc_gain'].values[0], adc_info['FGS1_adc_offset'].values[0])","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:05.830498Z","iopub.execute_input":"2024-09-02T21:46:05.830779Z","iopub.status.idle":"2024-09-02T21:46:06.298582Z","shell.execute_reply.started":"2024-09-02T21:46:05.830755Z","shell.execute_reply":"2024-09-02T21:46:06.297601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(restored_signal.iloc[0].values.reshape(32, 356), cmap='viridis')\nplt.colorbar()\nplt.title('Restored Signal Frame')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:06.300303Z","iopub.execute_input":"2024-09-02T21:46:06.301024Z","iopub.status.idle":"2024-09-02T21:46:06.785075Z","shell.execute_reply.started":"2024-09-02T21:46:06.300971Z","shell.execute_reply":"2024-09-02T21:46:06.783995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modelling","metadata":{}},{"cell_type":"code","source":"class ExoplanetDataset(Dataset):\n    def __init__(self, signal_data, labels):\n        self.signal_data = signal_data\n        self.labels = labels\n\n    def __len__(self):\n        return len(self.signal_data)\n\n    def __getitem__(self, idx):\n        x = self.signal_data[idx]\n        y = self.labels[idx]\n        return x, y\n","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:06.786486Z","iopub.execute_input":"2024-09-02T21:46:06.786838Z","iopub.status.idle":"2024-09-02T21:46:06.794339Z","shell.execute_reply.started":"2024-09-02T21:46:06.786806Z","shell.execute_reply":"2024-09-02T21:46:06.793073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_data(planet_id):\n    signal_data = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/train/{planet_id}/AIRS-CH0_signal.parquet')\n    adc_info = train_adc_info[train_adc_info['planet_id'] == planet_id]\n    restored_signal = restore_dynamic_range(signal_data, adc_info['FGS1_adc_gain'].values[0], adc_info['FGS1_adc_offset'].values[0])\n    return restored_signal\n","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:06.795911Z","iopub.execute_input":"2024-09-02T21:46:06.796292Z","iopub.status.idle":"2024-09-02T21:46:06.810813Z","shell.execute_reply.started":"2024-09-02T21:46:06.796259Z","shell.execute_reply":"2024-09-02T21:46:06.809944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"planet_id = 2633183716\nrestored_signal = prepare_data(planet_id)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:06.812081Z","iopub.execute_input":"2024-09-02T21:46:06.812418Z","iopub.status.idle":"2024-09-02T21:46:08.489313Z","shell.execute_reply.started":"2024-09-02T21:46:06.812384Z","shell.execute_reply":"2024-09-02T21:46:08.488392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Length of restored_signal: {len(restored_signal)}\")\nprint(f\"Length of train_labels: {len(train_labels)}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:08.490678Z","iopub.execute_input":"2024-09-02T21:46:08.491313Z","iopub.status.idle":"2024-09-02T21:46:08.49726Z","shell.execute_reply.started":"2024-09-02T21:46:08.491276Z","shell.execute_reply":"2024-09-02T21:46:08.496239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signal_length = len(restored_signal)\nlabel_length = train_labels.shape[0]\n","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:08.498615Z","iopub.execute_input":"2024-09-02T21:46:08.504268Z","iopub.status.idle":"2024-09-02T21:46:08.510466Z","shell.execute_reply.started":"2024-09-02T21:46:08.504216Z","shell.execute_reply":"2024-09-02T21:46:08.509287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if signal_length > label_length:\n    restored_signal = restored_signal[:label_length]\nelif label_length > signal_length:\n    train_labels = train_labels.iloc[:signal_length]\n\nprint(f\"Adjusted length of restored_signal: {len(restored_signal)}\")\nprint(f\"Adjusted length of train_labels: {len(train_labels)}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:08.511792Z","iopub.execute_input":"2024-09-02T21:46:08.512117Z","iopub.status.idle":"2024-09-02T21:46:08.524914Z","shell.execute_reply.started":"2024-09-02T21:46:08.512087Z","shell.execute_reply":"2024-09-02T21:46:08.523686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = ExoplanetDataset(restored_signal.values.reshape(-1, 1, 32, 356), train_labels.iloc[:, 1:].values)\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:08.526042Z","iopub.execute_input":"2024-09-02T21:46:08.526349Z","iopub.status.idle":"2024-09-02T21:46:08.537415Z","shell.execute_reply.started":"2024-09-02T21:46:08.526321Z","shell.execute_reply":"2024-09-02T21:46:08.536593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### I will build more complex model in future updates","metadata":{}},{"cell_type":"code","source":"class ExoplanetModel(nn.Module):\n    def __init__(self):\n        super(ExoplanetModel, self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, kernel_size=3, padding=1)\n        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)\n        \n        # Calculate the size after convolutions\n        self.conv_output_size = 64 * 32 * 356\n        \n        self.fc1 = nn.Linear(self.conv_output_size, 128)\n        self.fc2 = nn.Linear(128, 283)  # 283 wavelengths\n\n    def forward(self, x):\n        x = torch.relu(self.conv1(x))\n        x = torch.relu(self.conv2(x))\n        x = x.view(x.size(0), -1)  # Flatten the tensor\n        x = torch.relu(self.fc1(x))\n        x = self.fc2(x)\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:08.538647Z","iopub.execute_input":"2024-09-02T21:46:08.538982Z","iopub.status.idle":"2024-09-02T21:46:08.548528Z","shell.execute_reply.started":"2024-09-02T21:46:08.538953Z","shell.execute_reply":"2024-09-02T21:46:08.547395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ExoplanetModel()\ncriterion = nn.MSELoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:08.550192Z","iopub.execute_input":"2024-09-02T21:46:08.550554Z","iopub.status.idle":"2024-09-02T21:46:10.605213Z","shell.execute_reply.started":"2024-09-02T21:46:08.550524Z","shell.execute_reply":"2024-09-02T21:46:10.604401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# num_epochs = 10\n# for epoch in range(num_epochs):\n#     model.train()\n#     total_loss = 0\n#     for x, y in train_loader:\n#         x, y = x.float(), y.float()\n#         optimizer.zero_grad()\n#         outputs = model(x)\n#         loss = criterion(outputs, y)\n#         loss.backward()\n#         optimizer.step()\n#         total_loss += loss.item()\n#     print(f'Epoch {epoch+1}/{num_epochs}, Loss: {total_loss/len(train_loader)}')\n\n# torch.save(model.state_dict(), 'exoplanet_model.pth')","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:10.60634Z","iopub.execute_input":"2024-09-02T21:46:10.606766Z","iopub.status.idle":"2024-09-02T21:46:10.611355Z","shell.execute_reply.started":"2024-09-02T21:46:10.606739Z","shell.execute_reply":"2024-09-02T21:46:10.610299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ExoplanetModel()\nmodel_path = '/kaggle/input/exoplanet-model/pytorch/default/1/exoplanet_model.pth'\nmodel.load_state_dict(torch.load(model_path))\nmodel.eval()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:10.612439Z","iopub.execute_input":"2024-09-02T21:46:10.612715Z","iopub.status.idle":"2024-09-02T21:46:16.375882Z","shell.execute_reply.started":"2024-09-02T21:46:10.612687Z","shell.execute_reply":"2024-09-02T21:46:16.374746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_test_data(planet_id):\n    signal_data = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/test/{planet_id}/AIRS-CH0_signal.parquet')\n    adc_info = test_adc_info[test_adc_info['planet_id'] == planet_id]  # Assuming test_adc_info is available\n    restored_signal = restore_dynamic_range(signal_data, adc_info['FGS1_adc_gain'].values[0], adc_info['FGS1_adc_offset'].values[0])\n    return restored_signal\n\nplanet_id_test = 499191466 \nrestored_signal_test = prepare_test_data(planet_id_test)\nrestored_signal_test","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:16.377157Z","iopub.execute_input":"2024-09-02T21:46:16.377455Z","iopub.status.idle":"2024-09-02T21:46:18.647959Z","shell.execute_reply.started":"2024-09-02T21:46:16.377424Z","shell.execute_reply":"2024-09-02T21:46:18.647048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"planet_id_test = 499191466  \nrestored_signal_test = prepare_test_data(planet_id_test)\n\nrestored_signal_test = restored_signal_test.values.reshape(-1, 1, 32, 356)\n\ntest_dataset = ExoplanetDataset(restored_signal_test, np.zeros((restored_signal_test.shape[0], 283)))  # Dummy labels\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:18.649197Z","iopub.execute_input":"2024-09-02T21:46:18.649577Z","iopub.status.idle":"2024-09-02T21:46:20.006377Z","shell.execute_reply.started":"2024-09-02T21:46:18.649541Z","shell.execute_reply":"2024-09-02T21:46:20.005542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate_model(model, data_loader):\n    model.eval()\n    total_gll = 0\n    with torch.no_grad():\n        for x, _ in data_loader: \n            x = x.float()\n            outputs = model(x).numpy()\n            y_true = np.zeros_like(outputs)  \n            sigma_user = np.ones_like(outputs) * 1e-5 \n            gll = -0.5 * (np.log(2 * np.pi) + np.log(sigma_user**2) + ((y_true - outputs)**2 / sigma_user**2))\n            total_gll += gll.sum()\n    return total_gll","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:20.007354Z","iopub.execute_input":"2024-09-02T21:46:20.007623Z","iopub.status.idle":"2024-09-02T21:46:20.013947Z","shell.execute_reply.started":"2024-09-02T21:46:20.007599Z","shell.execute_reply":"2024-09-02T21:46:20.013078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gll_value = evaluate_model(model, test_loader)\nprint(f'Gaussian Log-Likelihood: {gll_value}')","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:46:20.015061Z","iopub.execute_input":"2024-09-02T21:46:20.015321Z","iopub.status.idle":"2024-09-02T21:48:21.078273Z","shell.execute_reply.started":"2024-09-02T21:46:20.015299Z","shell.execute_reply":"2024-09-02T21:48:21.077306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_spectra(y_true, y_pred, wavelengths):\n    plt.figure(figsize=(10, 6))\n    if y_true is not None:\n        plt.plot(wavelengths, y_true, label='True Spectrum', linestyle='--', color='blue')\n    plt.plot(wavelengths, y_pred, label='Predicted Spectrum', linestyle='-', color='red')\n    plt.xlabel('Wavelength')\n    plt.ylabel('Intensity')\n    plt.title('Exoplanet Atmospheric Spectrum')\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:48:21.079495Z","iopub.execute_input":"2024-09-02T21:48:21.07987Z","iopub.status.idle":"2024-09-02T21:48:21.086236Z","shell.execute_reply.started":"2024-09-02T21:48:21.079843Z","shell.execute_reply":"2024-09-02T21:48:21.08507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x, _ in test_loader:\n    x = x.float()\n    outputs = model(x)\n    y_pred = outputs.detach().numpy()\n    wavelengths = np.arange(283)\n    plot_spectra(None, y_pred[0], wavelengths)\n    break","metadata":{"execution":{"iopub.status.busy":"2024-09-02T21:48:21.08745Z","iopub.execute_input":"2024-09-02T21:48:21.087777Z","iopub.status.idle":"2024-09-02T21:48:21.768277Z","shell.execute_reply.started":"2024-09-02T21:48:21.087752Z","shell.execute_reply":"2024-09-02T21:48:21.767389Z"},"trusted":true},"execution_count":null,"outputs":[]}]}