{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":9189588,"sourceType":"datasetVersion","datasetId":5555090}],"dockerImageVersionId":30746,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":1087.424982,"end_time":"2024-08-03T13:00:18.164826","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-08-03T12:42:10.739844","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Ariel Data Challenge 2024: inference\n\nThe original version of the Train code is at:  [ADC24 Intro training](https://www.kaggle.com/code/ambrosm/adc24-intro-training).\nThe original version of the infer code is at: [ADC24 Intro inference](https://www.kaggle.com/code/ambrosm/adc24-intro-inference).\n\nI have added some new features, which resulted in a slight performance improvement(LB:0.388), \n\nThis is the inference code。","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.009657,"end_time":"2024-08-03T12:42:14.074227","exception":false,"start_time":"2024-08-03T12:42:14.06457","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport scipy.stats\nfrom tqdm import tqdm\nimport pickle\n\nfrom sklearn.linear_model import Ridge\n","metadata":{"_kg_hide-input":true,"papermill":{"duration":3.023083,"end_time":"2024-08-03T12:42:17.107326","exception":false,"start_time":"2024-08-03T12:42:14.084243","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-17T06:27:57.931854Z","iopub.execute_input":"2024-08-17T06:27:57.932258Z","iopub.status.idle":"2024-08-17T06:28:01.762038Z","shell.execute_reply.started":"2024-08-17T06:27:57.932226Z","shell.execute_reply":"2024-08-17T06:28:01.760866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"directory = \"/kaggle/input/test-001/\"\n\n#exec(open(directory + 'f_read_and_preprocess.py', 'r').read())\n#exec(open(directory + 'a_read_and_preprocess.py', 'r').read())\n#exec(open(directory + 'feature_engineering.py', 'r').read())\n#exec(open(directory + 'postprocessing.py', 'r').read())","metadata":{"_kg_hide-input":false,"papermill":{"duration":1067.258124,"end_time":"2024-08-03T13:00:08.197814","exception":false,"start_time":"2024-08-03T12:42:20.93969","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-17T06:28:07.255987Z","iopub.execute_input":"2024-08-17T06:28:07.256718Z","iopub.status.idle":"2024-08-17T06:28:07.26289Z","shell.execute_reply.started":"2024-08-17T06:28:07.256669Z","shell.execute_reply":"2024-08-17T06:28:07.261246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#f_read_and_preprocess\n\ndef f_read_and_preprocess(dataset, adc_info, planet_ids):\n    \"\"\"Read the FGS1 files for all planet_ids and extract the time series.\n    \n    Parameters\n    dataset: 'train' or 'test'\n    adc_info: metadata dataframe, either train_adc_info or test_adc_info\n    planet_ids: list of planet ids\n    \n    Returns\n    dataframe with one row per planet_id and 67500 values per row\n    \n    \"\"\"\n    f_raw_train = np.full((len(planet_ids), 67500), np.nan, dtype=np.float32)\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        f_signal = pl.read_parquet(f'../input/ariel-data-challenge-2024/{dataset}/{planet_id}/FGS1_signal.parquet')\n        mean_signal = f_signal.cast(pl.Int32).sum_horizontal().cast(pl.Float32).to_numpy() / 1024 # mean over the 32*32 pixels\n        net_signal = mean_signal[1::2] - mean_signal[0::2]\n        f_raw_train[i] = net_signal\n    return f_raw_train","metadata":{"execution":{"iopub.status.busy":"2024-08-17T06:28:11.731162Z","iopub.execute_input":"2024-08-17T06:28:11.731629Z","iopub.status.idle":"2024-08-17T06:28:11.741524Z","shell.execute_reply.started":"2024-08-17T06:28:11.73159Z","shell.execute_reply":"2024-08-17T06:28:11.740119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# a_read_and_preprocess\ndef a_read_and_preprocess(dataset, adc_info, planet_ids):\n    \"\"\"Read the AIRS-CH0 files for all planet_ids and extract the time series.\n    \n    Parameters\n    dataset: 'train' or 'test'\n    adc_info: metadata dataframe, either train_adc_info or test_adc_info\n    planet_ids: list of planet ids\n    \n    Returns\n    dataframe with one row per planet_id and 5625 values per row\n    \n    \"\"\"\n    a_raw_train = np.full((len(planet_ids), 5625), np.nan, dtype=np.float32)\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        signal = pl.read_parquet(f'../input/ariel-data-challenge-2024/{dataset}/{planet_id}/AIRS-CH0_signal.parquet')\n        mean_signal = signal.cast(pl.Int32).sum_horizontal().cast(pl.Float32).to_numpy() / (32*356) # mean over the 32*356 pixels\n        net_signal = mean_signal[1::2] - mean_signal[0::2]\n        a_raw_train[i] = net_signal\n    return a_raw_train","metadata":{"execution":{"iopub.status.busy":"2024-08-17T06:28:14.65448Z","iopub.execute_input":"2024-08-17T06:28:14.654904Z","iopub.status.idle":"2024-08-17T06:28:14.663629Z","shell.execute_reply.started":"2024-08-17T06:28:14.654853Z","shell.execute_reply":"2024-08-17T06:28:14.662491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature_engineering\ndef feature_engineering(f_raw, a_raw):\n    \"\"\"Create a dataframe with two features from the raw data.\n    \n    Parameters:\n    f_raw: ndarray of shape (n_planets, 67500)\n    a_raw: ndarray of shape (n_planets, 5625)\n    \n    Return value:\n    df: DataFrame of shape (n_planets, 2)\n    \"\"\"\n    obscured = f_raw[:, 23500:44000].mean(axis=1)\n    unobscured = (f_raw[:, :20500].mean(axis=1) + f_raw[:, 47000:].mean(axis=1)) / 2\n    f_relative_reduction = (unobscured - obscured) / unobscured\n    \n    half_obscured1 = f_raw[:, 20500:23500].mean(axis=1)\n    half_obscured2 = f_raw[:, 44000:47000].mean(axis=1)\n    f_half_reduction1 = (unobscured - half_obscured1) / unobscured\n    f_half_reduction2 = (unobscured - half_obscured2) / unobscured\n    \n    obscured = a_raw[:, 1958:3666].mean(axis=1)\n    unobscured = (a_raw[:, :1708].mean(axis=1) + a_raw[:, 3916:].mean(axis=1)) / 2\n    a_relative_reduction = (unobscured - obscured) / unobscured\n    \n    half_obscured1 = a_raw[:, 1708:1958].mean(axis=1)\n    half_obscured2 = a_raw[:, 3666:3916].mean(axis=1)\n    a_half_reduction1 = (unobscured - half_obscured1) / unobscured\n    a_half_reduction2 = (unobscured - half_obscured2) / unobscured\n\n    df = pd.DataFrame({'a_relative_reduction': a_relative_reduction,\n                       'f_relative_reduction': f_relative_reduction,\n                      'f_half_reduction1': f_half_reduction1,\n                       'f_half_reduction2': f_half_reduction2,\n                       'a_half_reduction1': a_half_reduction1,\n                       'a_half_reduction2': a_half_reduction2\n                      \n                      })\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-08-17T06:28:17.599302Z","iopub.execute_input":"2024-08-17T06:28:17.600355Z","iopub.status.idle":"2024-08-17T06:28:17.612911Z","shell.execute_reply.started":"2024-08-17T06:28:17.60031Z","shell.execute_reply":"2024-08-17T06:28:17.611521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 'postprocessing\ndef postprocessing(pred_array, index, sigma_pred):\n    \"\"\"Create a submission dataframe from its components\n    \n    Parameters:\n    pred_array: ndarray of shape (n_samples, 283)\n    index: pandas.Index of length n_samples with name 'planet_id'\n    sigma_pred: float\n    \n    Return value:\n    df: DataFrame of shape (n_samples, 566) with planet_id as index\n    \"\"\"\n    return pd.concat([pd.DataFrame(pred_array.clip(0, None), index=index, columns=wavelengths.columns),\n                      pd.DataFrame(sigma_pred, index=index, columns=[f\"sigma_{i}\" for i in range(1, 284)])],\n                     axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T06:28:22.157315Z","iopub.execute_input":"2024-08-17T06:28:22.157809Z","iopub.status.idle":"2024-08-17T06:28:22.166343Z","shell.execute_reply.started":"2024-08-17T06:28:22.157772Z","shell.execute_reply":"2024-08-17T06:28:22.164729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"People have been asking how to choose a good value for sigma_pred. As explained in [Understanding the competition metric](https://www.kaggle.com/competitions/ariel-data-challenge-2024/discussion/528114), with sigma_pred we indicate what root mean squared error (rmse) we expect for our test predictions.\n\nThe training data cover planets of only two stars (stars 0 and 1), but the test data include planets of other stars.\n\nThis leads to the following recipe:\n- For known stars (stars 0 and 1), we expect the test rmse to be equal to our cross-validation rmse, i.e. we predict the out-of-fold rmse of our model (0.000293 as shown in the training notebook).\n- For unknown stars, the prediction error can only be higher. We thus predict a higher value (0.001 in this notebook).\n\n如何选择一个好的 sigma_pred 值。正如在理解竞赛指标中解释的，通过 sigma_pred 我们表示我们期望的测试预测的均方根误差（rmse）。\n\n训练数据只涵盖了两颗恒星（恒星0和1）的行星，但测试数据包括其他恒星的行星。\n\n这导致了以下问题：\n\n对于已知恒星（恒星0和1），我们期望测试 rmse 等于我们的交叉验证 rmse，即我们预测模型的 out-of-fold rmse（如训练所示，为0.000293）。\n对于未知恒星，预测误差只能更高。因此我们预测一个更高的值（目前为0.001）。","metadata":{}},{"cell_type":"code","source":"# Load the data\nwavelengths = pd.read_csv('../input/ariel-data-challenge-2024/wavelengths.csv')\ntest_adc_info = pd.read_csv('../input/ariel-data-challenge-2024/test_adc_info.csv',\n                           index_col='planet_id')\nf_raw_test = f_read_and_preprocess('test', test_adc_info, test_adc_info.index)\na_raw_test = a_read_and_preprocess('test', test_adc_info, test_adc_info.index)\ntest = feature_engineering(f_raw_test, a_raw_test)\n\n# Load the model\nwith open(directory + 'model.pickle', 'rb') as f:\n    model = pickle.load(f)\nwith open(directory + 'sigma_pred.pickle', 'rb') as f:\n    sigma_pred = pickle.load(f)\n    \n# Predict\ntest_pred = model.predict(test)\n\n# Package into submission file\nsub_df = postprocessing(test_pred,\n                        test_adc_info.index,\n                        sigma_pred=np.tile(np.where(test_adc_info[['star']] <= 1, sigma_pred * 0.95, 0.001), (1, 283)))\ndisplay(sub_df)\nsub_df.to_csv('submission.csv')\n#!head submission.csv","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-08-17T06:28:26.278392Z","iopub.execute_input":"2024-08-17T06:28:26.278865Z","iopub.status.idle":"2024-08-17T06:28:29.518606Z","shell.execute_reply.started":"2024-08-17T06:28:26.278832Z","shell.execute_reply":"2024-08-17T06:28:29.517387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}