{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":101849,"databundleVersionId":12846694,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#### This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-06-28T15:15:48.372626Z","iopub.execute_input":"2025-06-28T15:15:48.372989Z","iopub.status.idle":"2025-06-28T15:16:09.937072Z","shell.execute_reply.started":"2025-06-28T15:15:48.372964Z","shell.execute_reply":"2025-06-28T15:16:09.935882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Load data\ntrain = pd.read_csv(\"/kaggle/input/ariel-data-challenge-2025/train.csv\")\nsample_sub = pd.read_csv(\"/kaggle/input/ariel-data-challenge-2025/sample_submission.csv\")\n\n# Print shapes\nprint(\"Train shape:\", train.shape)\nprint(\"Sample submission shape:\", sample_sub.shape)\n\n# Peek at train\ntrain.head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T15:16:09.938686Z","iopub.execute_input":"2025-06-28T15:16:09.939144Z","iopub.status.idle":"2025-06-28T15:16:10.201972Z","shell.execute_reply.started":"2025-06-28T15:16:09.939095Z","shell.execute_reply":"2025-06-28T15:16:10.200194Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Compute mean spectrum from training set\nspectrum_columns = [col for col in train.columns if col.startswith(\"wl_\")]\nmean_spectrum = train[spectrum_columns].mean().values\n\n# Step 2: Create predictions for each test planet\n# Extract test planet_ids from sample_submission\ntest_planet_ids = sample_sub[\"planet_id\"]\n\n# Step 3: Build the prediction DataFrame\npred_df = pd.DataFrame()\npred_df[\"planet_id\"] = test_planet_ids\n\n# Add 283 spectral predictions (mean_spectrum)\nfor i, wl in enumerate(spectrum_columns):\n    pred_df[wl] = mean_spectrum[i]\n\n# Add 283 uncertainty values — let's use a constant 0.01 for now\nfor i, wl in enumerate(spectrum_columns):\n    pred_df[f\"{wl}_uncertainty\"] = 0.01\n\n# Step 4: Save submission file\npred_df.to_csv(\"/kaggle/working/submission.csv\", index=False)\npred_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T15:16:10.203426Z","iopub.execute_input":"2025-06-28T15:16:10.203783Z","iopub.status.idle":"2025-06-28T15:16:10.453204Z","shell.execute_reply.started":"2025-06-28T15:16:10.203747Z","shell.execute_reply":"2025-06-28T15:16:10.452202Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ariel Data Challenge 2025 - End-to-End Notebook\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pickle\nfrom glob import glob\nfrom tqdm import tqdm\n\nfrom sklearn.linear_model import Ridge\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.model_selection import train_test_split\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T15:16:10.454149Z","iopub.execute_input":"2025-06-28T15:16:10.454425Z","iopub.status.idle":"2025-06-28T15:16:14.091565Z","shell.execute_reply.started":"2025-06-28T15:16:10.454403Z","shell.execute_reply":"2025-06-28T15:16:14.090157Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Data\n\ntrain_df = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/train.csv', index_col='planet_id')\nwavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/wavelengths.csv')\ntrain_star_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/train_star_info.csv')\ntrain_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/adc_info.csv')\n\ndef f_read_and_preprocess(dataset, planet_ids):\n    f_raw = np.full((len(planet_ids), 67500), np.nan, dtype=np.float32)\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        f_signal = pl.read_parquet(f'/kaggle/input/ariel-data-challenge-2025/{dataset}/{planet_id}/FGS1_signal_0.parquet')\n        mean_signal = f_signal.cast(pl.Int32).sum_horizontal().cast(pl.Float32).to_numpy() / 1024\n        net_signal = mean_signal[1::2] - mean_signal[0::2]\n        f_raw[i] = net_signal\n    return f_raw\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T15:16:14.094536Z","iopub.execute_input":"2025-06-28T15:16:14.095036Z","iopub.status.idle":"2025-06-28T15:16:14.252242Z","shell.execute_reply.started":"2025-06-28T15:16:14.095008Z","shell.execute_reply":"2025-06-28T15:16:14.251085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def a_read_and_preprocess(dataset, planet_ids):\n    a_raw = np.full((len(planet_ids), 5625), np.nan, dtype=np.float32)\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        signal = pl.read_parquet(f'/kaggle/input/ariel-data-challenge-2025/{dataset}/{planet_id}/AIRS-CH0_signal_0.parquet')\n        mean_signal = signal.cast(pl.Int32).sum_horizontal().cast(pl.Float32).to_numpy() / (32*356)\n        net_signal = mean_signal[1::2] - mean_signal[0::2]\n        a_raw[i] = net_signal\n    return a_raw\n\nplanet_ids = train_df.index\nf_raw_train = f_read_and_preprocess('train', planet_ids)\na_raw_train = a_read_and_preprocess('train', planet_ids)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T15:16:14.253065Z","iopub.execute_input":"2025-06-28T15:16:14.253369Z","iopub.status.idle":"2025-06-28T15:49:27.520176Z","shell.execute_reply.started":"2025-06-28T15:16:14.253348Z","shell.execute_reply":"2025-06-28T15:49:27.516852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Load training metadata and labels\ntrain_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/train_star_info.csv', index_col='planet_id')\ntrain_labels = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/train.csv', index_col='planet_id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T15:49:27.525331Z","iopub.execute_input":"2025-06-28T15:49:27.526004Z","iopub.status.idle":"2025-06-28T15:49:27.80073Z","shell.execute_reply.started":"2025-06-28T15:49:27.525933Z","shell.execute_reply":"2025-06-28T15:49:27.799782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile f_read_and_preprocess.py\nimport numpy as np\nimport polars as pl\nfrom tqdm import tqdm\n\ndef f_read_and_preprocess(dataset, adc_info, planet_ids):\n    \"\"\"Read the FGS1 files for all planet_ids and extract the time series.\n    \n    Parameters\n    dataset: 'train' or 'test'\n    adc_info: metadata dataframe\n    planet_ids: list of planet ids\n    \n    Returns\n    ndarray with one row per planet_id and 67500 values per row\n    \"\"\"\n    f_raw = np.full((len(planet_ids), 67500), np.nan, dtype=np.float32)\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        f_signal = pl.read_parquet(f'/kaggle/input/ariel-data-challenge-2025/{dataset}/{planet_id}/FGS1_signal_0.parquet')\n        mean_signal = f_signal.cast(pl.Int32).sum_horizontal().cast(pl.Float32).to_numpy() / 1024\n        net_signal = mean_signal[1::2] - mean_signal[0::2]\n        f_raw[i] = net_signal\n    return f_raw","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T15:49:27.802003Z","iopub.execute_input":"2025-06-28T15:49:27.802295Z","iopub.status.idle":"2025-06-28T15:49:27.815296Z","shell.execute_reply.started":"2025-06-28T15:49:27.802273Z","shell.execute_reply":"2025-06-28T15:49:27.81418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile a_read_and_preprocess.py\nimport numpy as np\nimport polars as pl\nfrom tqdm import tqdm\n\ndef a_read_and_preprocess(dataset, adc_info, planet_ids):\n    \"\"\"Read the AIRS-CH0 files for all planet_ids and extract the time series.\n    \n    Parameters\n    dataset: 'train' or 'test'\n    adc_info: metadata dataframe\n    planet_ids: list of planet ids\n    \n    Returns\n    ndarray with one row per planet_id and 5625 values per row\n    \"\"\"\n    a_raw = np.full((len(planet_ids), 5625), np.nan, dtype=np.float32)\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        signal = pl.read_parquet(f'/kaggle/input/ariel-data-challenge-2025/{dataset}/{planet_id}/AIRS-CH0_signal_0.parquet')\n        mean_signal = signal.cast(pl.Int32).sum_horizontal().cast(pl.Float32).to_numpy() / (32 * 356)\n        net_signal = mean_signal[1::2] - mean_signal[0::2]\n        a_raw[i] = net_signal\n    return a_raw","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T15:49:27.816735Z","iopub.execute_input":"2025-06-28T15:49:27.817314Z","iopub.status.idle":"2025-06-28T15:49:27.842766Z","shell.execute_reply.started":"2025-06-28T15:49:27.817286Z","shell.execute_reply":"2025-06-28T15:49:27.841352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(f_raw, a_raw, n_bins=75):\n    f_feat = f_raw.reshape(f_raw.shape[0], n_bins, -1).mean(axis=2)\n    a_feat = a_raw.reshape(a_raw.shape[0], n_bins, -1).mean(axis=2)\n    return np.concatenate([f_feat, a_feat], axis=1)\n\nexec(open('f_read_and_preprocess.py', 'r').read())\nexec(open('a_read_and_preprocess.py', 'r').read())\n\nf_raw_train = f_read_and_preprocess('train', train_adc_info, train_labels.index)\na_raw_train = a_read_and_preprocess('train', train_adc_info, train_labels.index)\n\nX = feature_engineering(f_raw_train, a_raw_train)\ny = train_df.values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T15:49:27.844059Z","iopub.execute_input":"2025-06-28T15:49:27.844379Z","iopub.status.idle":"2025-06-28T16:22:48.85092Z","shell.execute_reply.started":"2025-06-28T15:49:27.844357Z","shell.execute_reply":"2025-06-28T16:22:48.849245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)\n\nmodel = Ridge(alpha=0.1)\nmodel.fit(X_scaled, y)\ny_pred = model.predict(X_scaled)\n\nmse = mean_squared_error(y, y_pred)\nr2 = r2_score(y, y_pred)\nprint(f\"Train MSE: {mse:.6f} | R2: {r2:.6f}\")\n\nsigma_pred = 0.01  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:22:48.85215Z","iopub.execute_input":"2025-06-28T16:22:48.852541Z","iopub.status.idle":"2025-06-28T16:22:49.051301Z","shell.execute_reply.started":"2025-06-28T16:22:48.85251Z","shell.execute_reply":"2025-06-28T16:22:49.046392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save \n\nwith open('model.pickle', 'wb') as f:\n    pickle.dump(model, f)\nwith open('scaler.pickle', 'wb') as f:\n    pickle.dump(scaler, f)\nwith open('sigma_pred.pickle', 'wb') as f:\n    pickle.dump(sigma_pred, f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:22:49.055568Z","iopub.execute_input":"2025-06-28T16:22:49.056021Z","iopub.status.idle":"2025-06-28T16:22:49.073096Z","shell.execute_reply.started":"2025-06-28T16:22:49.055954Z","shell.execute_reply":"2025-06-28T16:22:49.072373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/test_star_info.csv', index_col='planet_id')\nsample_submission = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/sample_submission.csv', index_col='planet_id')\n\nf_raw_test = f_read_and_preprocess('test', test_adc_info, sample_submission.index)\na_raw_test = a_read_and_preprocess('test', test_adc_info, sample_submission.index)\n\nX_test = feature_engineering(f_raw_test, a_raw_test)\nX_test_scaled = scaler.transform(X_test)\ny_test_pred = model.predict(X_test_scaled)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:22:49.074822Z","iopub.execute_input":"2025-06-28T16:22:49.075394Z","iopub.status.idle":"2025-06-28T16:22:50.891568Z","shell.execute_reply.started":"2025-06-28T16:22:49.075368Z","shell.execute_reply":"2025-06-28T16:22:50.890405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def postprocessing(pred_array, index, sigma_pred):\n    columns = [f\"wl_{i+1}\" for i in range(pred_array.shape[1])]\n    df_pred = pd.DataFrame(pred_array.clip(0, None), index=index, columns=columns)\n    if np.isscalar(sigma_pred):\n        sigma_array = np.full_like(pred_array, sigma_pred)\n    else:\n        sigma_array = sigma_pred\n    df_sigma = pd.DataFrame(sigma_array, index=index, columns=[f\"{c}_uncertainty\" for c in columns])\n    return pd.concat([df_pred, df_sigma], axis=1)\n\nsubmission = postprocessing(y_test_pred, sample_submission.index, sigma_pred)\nsubmission.to_csv('submission.csv')\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:22:50.898076Z","iopub.execute_input":"2025-06-28T16:22:50.898487Z","iopub.status.idle":"2025-06-28T16:22:50.943041Z","shell.execute_reply.started":"2025-06-28T16:22:50.898462Z","shell.execute_reply":"2025-06-28T16:22:50.942095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}