{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":9433826,"sourceType":"datasetVersion","datasetId":5731720},{"sourceId":9658115,"sourceType":"datasetVersion","datasetId":5613665}],"dockerImageVersionId":30762,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install -U scikit-learn==1.5.2 -f ../input/sklearn1-5-2 --no-index","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:16:55.752499Z","iopub.execute_input":"2024-10-18T13:16:55.753003Z","iopub.status.idle":"2024-10-18T13:17:13.105587Z","shell.execute_reply.started":"2024-10-18T13:16:55.752954Z","shell.execute_reply":"2024-10-18T13:17:13.104508Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport glob \nimport random\nimport polars as pl\nimport gc\nimport pickle\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pytorch_lightning \nimport time\nfrom torchmetrics import MeanAbsoluteError\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport cupy as cp    \nfrom scipy.optimize import minimize\nfrom scipy import optimize\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pytorch_lightning\nfrom torchmetrics import MeanAbsoluteError\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom torch.optim import Adam\n\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.linear_model import LinearRegression\nimport pickle\n\n\ntorch.manual_seed(0)\nrandom.seed(0)\nnp.random.seed(0)\nt0 = time.time()\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nprint(np.__version__)\nprint(torch.__version__)\nprint(pytorch_lightning.__version__)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-18T13:17:13.107902Z","iopub.execute_input":"2024-10-18T13:17:13.108313Z","iopub.status.idle":"2024-10-18T13:17:21.315462Z","shell.execute_reply.started":"2024-10-18T13:17:13.108271Z","shell.execute_reply":"2024-10-18T13:17:21.314425Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DEBUG = False\nSTEP_BY_STEP = False","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:21.317173Z","iopub.execute_input":"2024-10-18T13:17:21.317784Z","iopub.status.idle":"2024-10-18T13:17:21.323874Z","shell.execute_reply.started":"2024-10-18T13:17:21.317738Z","shell.execute_reply":"2024-10-18T13:17:21.322925Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def create_fake_submission_file():\n#     wavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')\n#     fake_submit = pd.DataFrame(index = index, columns=list(wavelengths.columns) + [c.replace(\"wl_\", \"sigma_\") for c in wavelengths.columns])\n#     fake_submit.fillna(0, inplace=True)\n#     fake_submit.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:21.326576Z","iopub.execute_input":"2024-10-18T13:17:21.326953Z","iopub.status.idle":"2024-10-18T13:17:21.350644Z","shell.execute_reply.started":"2024-10-18T13:17:21.326909Z","shell.execute_reply":"2024-10-18T13:17:21.349884Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path_folder = '/kaggle/input/ariel-data-challenge-2024/' # path to the folder containing the data\npath_out = './preprocessed_data/' # path to the folder to store the light data\noutput_dir = './preprocessed_data/' # path for the output directory\n\nif not os.path.exists(path_out):\n    os.makedirs(path_out)\n    print(f\"Directory {path_out} created.\")\nelse:\n    print(f\"Directory {path_out} already exists.\")","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:21.351816Z","iopub.execute_input":"2024-10-18T13:17:21.352177Z","iopub.status.idle":"2024-10-18T13:17:21.361528Z","shell.execute_reply.started":"2024-10-18T13:17:21.352135Z","shell.execute_reply":"2024-10-18T13:17:21.360702Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Pre-processing the data","metadata":{}},{"cell_type":"code","source":"def ADC_convert(signal, gain, offset):\n    signal /= gain\n    signal += offset\n    return signal\n\ndef clean_flat_dark_dead(signal, dead, flat, dark):\n    signal = (signal - dark ) / (flat - dark)\n    dead_mask = cp.asarray((dead == 1.0).reshape((-1,)))\n    signal[:, dead_mask] = cp.nan\n    return signal\n\ndef apply_linear_corr(linear_corr, signal):\n    result = linear_corr[5, :]\n    for i in range(4, -1, -1):\n        result = result * signal + linear_corr[i, :]\n    return result\n\n\ndef bin_obs(signal, binning):\n    signal_binned = cp.zeros((signal.shape[0] // binning, signal.shape[1], signal.shape[2]))\n    for i in range(signal.shape[0] // binning):\n        signal_binned[i, :, :] = cp.sum(signal[i * binning : (i + 1) * binning, :, :], axis=0)\n    return signal_binned\n\n\ndef load_signal_data(planet_id, dataset, instrument, adc_info, step_size=30):\n    precision = pl.Float32 # pl.Float64\n    binning = step_size if instrument == \"AIRS-CH0\" else step_size * 12  # 4642\n    img_size = (32, 356) if instrument == \"AIRS-CH0\" else (32, 32)\n    gain, offset = (\n        adc_info.loc[planet_id][f\"{instrument}_adc_gain\"],\n        adc_info.loc[planet_id][f\"{instrument}_adc_offset\"],\n    )\n    signal_file = f\"{path_folder}/{dataset}/{planet_id}/{instrument}_signal.parquet\"\n    calibration_file = f\"{path_folder}/{dataset}/{planet_id}/{instrument}_calibration\"\n    \n    signal = cp.array(pl.read_parquet(signal_file).cast(precision).to_numpy())\n\n    flat = cp.array(pl.read_parquet(f\"{calibration_file}/flat.parquet\").cast(precision).to_numpy().reshape((1, -1)))\n    dark = cp.array(pl.read_parquet(f\"{calibration_file}/dark.parquet\").cast(precision).to_numpy().reshape((1, -1)))\n    dead = cp.array(pl.read_parquet(f\"{calibration_file}/dead.parquet\").cast(precision).to_numpy().reshape((1, -1)))\n  \n    linear_corr = cp.array(\n        pl.read_parquet(f\"{calibration_file}/linear_corr.parquet\").cast(precision).to_numpy().reshape(6, -1)\n    )\n\n    signal = ADC_convert(signal, gain, offset)  # AIRS: dim=(11'250, 11'392)\n    signal = clean_flat_dark_dead(signal, dead, flat, dark)\n    signal = apply_linear_corr(linear_corr, signal) \n    signal = signal.reshape((signal.shape[0], img_size[0], img_size[1]))\n    signal = signal[1::2] - signal[0::2]    \n    signal = bin_obs(signal, binning)\n    \n    if instrument == \"AIRS-CH0\":\n        return cp.asnumpy(signal)[:, :, 39:321]\n    else:\n        return cp.asnumpy(signal)\n    \ndef generate_signal(mode, adc_info, instrument=\"FGS1\",):\n    assert instrument in [\"FGS1\", \"AIRS-CH0\"]\n    signal = []\n    for planet_id in tqdm(adc_info.index):\n        signal.append(load_signal_data(planet_id, mode, instrument, adc_info))\n    return np.stack(signal).transpose((0, 1, 3, 2))","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:21.362864Z","iopub.execute_input":"2024-10-18T13:17:21.363575Z","iopub.status.idle":"2024-10-18T13:17:21.381539Z","shell.execute_reply.started":"2024-10-18T13:17:21.363533Z","shell.execute_reply":"2024-10-18T13:17:21.380731Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if DEBUG:\n    files = glob.glob(os.path.join(path_folder + 'train/', '*/*'))\n    files = files[:150]  # 65 -> 16 exoplanets ; 150 -> 38 exoplanets \n    files = files[:25]\n    \n    adc_info = pd.read_csv(os.path.join(path_folder, 'train_adc_info.csv'))\n    adc_info.set_index(\"planet_id\", inplace=True)\n    \n    exoplanet_ids = list(set([int(c.split(\"/\")[-2]) for c in files]))\n    adc_info = adc_info[adc_info.index.isin(exoplanet_ids)]\n    print(exoplanet_ids)\n\nelse:\n    files = glob.glob(os.path.join(path_folder + 'test/', '*/*'))\n    \n    adc_info = pd.read_csv(os.path.join(path_folder, 'test_adc_info.csv'))\n    adc_info.set_index(\"planet_id\", inplace=True)\n\n# index = adc_info.index\n\nprint(len(files))\nprint(\"adc_info's shape:\", adc_info.shape)\nt1 = time.time()\n\nN = len(adc_info.index.unique())\n\nif DEBUG: print(f\"[{t1-t0:6.1f}s]  {N} exoplanets\")","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:21.382481Z","iopub.execute_input":"2024-10-18T13:17:21.382813Z","iopub.status.idle":"2024-10-18T13:17:23.64394Z","shell.execute_reply.started":"2024-10-18T13:17:21.382782Z","shell.execute_reply":"2024-10-18T13:17:23.642987Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_data(adc_info):\n    field = (\"test\", \"train\")[DEBUG]\n    data_train_FGS = generate_signal(field, adc_info, instrument=\"FGS1\")\n    data_train = generate_signal(field, adc_info, instrument=\"AIRS-CH0\")\n    print(\"shapes:\", data_train_FGS.shape, data_train.shape)\n    FGS_column = np.nanmean(data_train_FGS, axis = 2)\n    return data_train, FGS_column\n\n\nif DEBUG and STEP_BY_STEP: \n    data_train, FGS_column = preprocess_data(adc_info)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.645132Z","iopub.execute_input":"2024-10-18T13:17:23.645416Z","iopub.status.idle":"2024-10-18T13:17:23.651273Z","shell.execute_reply.started":"2024-10-18T13:17:23.645385Z","shell.execute_reply":"2024-10-18T13:17:23.650321Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Predictions","metadata":{}},{"cell_type":"code","source":"def get_pre_train_data(data_train, FGS_column):\n    fgs_summed = np.nanmean(FGS_column, axis = 2)\n    airs_summed = np.nanmean(data_train[:, :, :, 10:22], axis = 3)\n    pre_train = np.concatenate([fgs_summed[:, :, np.newaxis], airs_summed], axis=2)\n    return pre_train\n\n\nif DEBUG and STEP_BY_STEP: \n    pre_train = get_pre_train_data(data_train, FGS_column)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.652931Z","iopub.execute_input":"2024-10-18T13:17:23.653346Z","iopub.status.idle":"2024-10-18T13:17:23.66118Z","shell.execute_reply.started":"2024-10-18T13:17:23.653302Z","shell.execute_reply":"2024-10-18T13:17:23.66028Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SHOW_PLOT = False\n\ndef phase_detector(signal):\n    \n    MIN = np.argmin(signal[30:140])+30\n    signal1 = signal[:MIN ]\n    signal2 = signal[MIN :]\n\n    first_derivative1 = np.gradient(signal1)\n    first_derivative1 /= first_derivative1.max()\n    first_derivative2 = np.gradient(signal2)\n    first_derivative2 /= first_derivative2.max()\n\n    phase1 = np.argmin(first_derivative1)\n    phase2 = np.argmax(first_derivative2) + MIN\n\n    return phase1, phase2\n  \n\ndef best_ratio_search(signal, p1, p2, poylomial_order, delta=3):\n    DELTA = delta\n    def objective(s, signal):\n        \"\"\" on applique un facteur multiplicatif à la zone de transit => courbe y.\n            on cherche le facteur qui minimise l'écart entre le courbe 'scalée' y et un fit polynomial d'ordre 3.\n        \"\"\"\n        best_q = 1e10\n        \n        x = list(range(signal.shape[0]-DELTA*4))\n\n        length = signal.shape[0]\n        signal = np.array(list(zip(range(length), signal.tolist())))\n        signal = pd.DataFrame(signal[:, 1], index=signal[:, 0])\n        tmp = pd.concat([signal.iloc[:p1 - DELTA], signal.iloc[p1 + DELTA:p2-DELTA] * (1+s), signal.iloc[p2+DELTA:]])\n        x = list(tmp.index)\n        y = list(tmp[0].values)            \n        z = np.polyfit(x, y, deg=poylomial_order)\n        p = np.poly1d(z)\n        q = np.abs(p(x) - y).mean()\n        \n        if q < best_q :\n            best_q = q\n        \n        return q\n    \n    r = minimize(\n                objective,\n                [0.0001],   # première valeur de s ===> paramètre x\n                method= 'Nelder-Mead',\n                args = (signal)\n                  )\n    s = r.x[0]\n    return s\n\ntransit_limiting_timestamps = []\ndef get_ratio(features, pre_train):\n    \n    all_s = []\n    partial_s = []\n    for i in tqdm(range(pre_train.shape[0])):\n\n        # on prend une exoplenete et on fait la moyenne sur les longueur d'onde\n        # FGS est supprimé pour déterminer la période de transit\n        signal = pre_train[i,:,1:].mean(axis=1)   \n        # détermination des débuts / fin de transit\n        p1, p2 = phase_detector(signal)  \n        transit_limiting_timestamps.append([p1, p2])\n        \n        ###############################\n        # TO DEBUG\n        if SHOW_PLOT:\n            print(p1, p2)\n            plt.plot(signal) ; \n            plt.axvline(p1, color='r') ; plt.axvline(p2, color='r') ; \n            plt.grid()\n        ###############################\n        \n        # field=f'{poylomial_order}_{k}_{delta}_{NWAVE}'\n        ratios = {}\n        agg_features = []\n        for feature in features:\n            if feature.startswith(\"tot\") : \n                agg_features.append(feature)\n                continue\n                \n            poylomial_order, k, delta, NWAVE = map(int, feature.split(\"_\"))     \n            signal = pre_train[i,:,k-NWAVE:k+NWAVE].mean(axis=1)\n            field=f'{poylomial_order}_{k}_{delta}_{NWAVE}'\n            s = best_ratio_search(signal, p1, p2, poylomial_order, delta=delta)\n            ratios[field]= s \n        partial_s.append(ratios)\n        \n        ratios = {}\n        for feature in agg_features:\n            _, delta, order = feature.split(\"_\")\n            delta = int(delta.strip(\"delta=\"))\n            order = int(order.strip(\"order=\"))\n            # ratio avec sommation sur l'ensemble des longueurs d'onde\n            signal = pre_train[i,:,1:].mean(axis=1)\n            s = best_ratio_search(signal, p1, p2, poylomial_order, delta=delta)\n            ratios[feature] = s\n        all_s.append(ratios)\n            \n        # print(s, np.mean(ratios))\n        \n        ###############################\n        # TO DEBUG\n        if SHOW_PLOT:\n            DELTA = 3\n            print(\"ratio:\", s)\n            length = signal.shape[0]\n            signal = np.array(list(zip(range(length), signal.tolist())))\n            signal = pd.DataFrame(signal[:, 1], index=signal[:, 0])\n            tmp = pd.concat([signal.iloc[:p1 - DELTA], signal.iloc[p1 + DELTA:p2-DELTA] * (1+s), signal.iloc[p2+DELTA:]])\n            x = list(tmp.index)\n            y = list(tmp[0].values)\n            z = np.polyfit(x, y, deg=3)\n            p = np.poly1d(z)\n            plt.plot(x, p(x))\n            plt.plot(tmp.loc[p1:p2].index.values, tmp.loc[p1:p2][0].values)\n            plt.title(f\"target = {target_mean[i]:8.5f} / pred = {s:8.5f} / ratio = {s / target_mean[i] : 7.3f}\")\n            plt.show()\n        ###############################\n    \n        \n    # all_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))        \n    return all_s, partial_s, transit_limiting_timestamps\n\n\n\ndef get_fluxes(model_list, adc_info, pre_train):\n    \n       \n    list_of_features = []\n    for feature_names_file, _ in model_list:\n        feature_names = pickle.load(open(f\"../input/adc24-checkpoints/{feature_names_file}\", 'rb'))\n        list_of_features.extend(feature_names)\n    \n    feature_names = list(set(list_of_features))\n\n\n    total_ratio, partial_ratio, transit_limiting_timestamps = get_ratio(feature_names, pre_train)\n\n    \n    features = pd.DataFrame()\n    if len(total_ratio) > 0:\n        features = pd.DataFrame(total_ratio, index=adc_info.index)\n    \n    if len(partial_ratio) > 0:\n        partial = pd.DataFrame(partial_ratio, index=adc_info.index)\n        if features.empty:\n            features = partial\n        else:\n            features = features.join(partial)\n\n    return features\n\n\nif DEBUG and STEP_BY_STEP: \n    model_list = [\n        (\"features_model8_2.pkl\", \"reg_model8_fold<NUMBER>_2.pkl\"),\n        (\"features_model8_3.pkl\", \"reg_model8_fold<NUMBER>_3.pkl\"),\n        (\"features_model8_4.pkl\", \"reg_model8_fold<NUMBER>_4.pkl\"),\n        (\"features_model8_5.pkl\", \"reg_model8_fold<NUMBER>_5.pkl\"),\n        (\"features_model8_6.pkl\", \"reg_model8_fold<NUMBER>_6.pkl\"),\n        (\"features_model8_7.pkl\", \"reg_model8_fold<NUMBER>_7.pkl\"),\n        (\"features_model8_8.pkl\", \"reg_model8_fold<NUMBER>_8.pkl\"),\n    ]\n    features = get_fluxes(model_list, adc_info, pre_train)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.665787Z","iopub.execute_input":"2024-10-18T13:17:23.666087Z","iopub.status.idle":"2024-10-18T13:17:23.695453Z","shell.execute_reply.started":"2024-10-18T13:17:23.666056Z","shell.execute_reply":"2024-10-18T13:17:23.694684Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_mean_predictions_1(model_list, features):\n    \n    aggregated_predictions = []\n    for feature_names_file, model_name in model_list:\n        feature_names = pickle.load(open(f\"../input/adc24-checkpoints/{feature_names_file}\", 'rb'))\n        Nfold = 5\n        predictions = np.zeros((features.shape[0], Nfold))\n        for fold in range(Nfold):\n            _name = model_name.replace(\"<NUMBER>\", str(fold)) \n            reg = pickle.load(open(f\"../input/adc24-checkpoints/{_name}\", 'rb'))\n            predictions[:, fold] = reg.predict(features[feature_names])\n        all_s = predictions.mean(axis=1)\n        aggregated_predictions.append(all_s)\n\n    all_s = np.array(aggregated_predictions).mean(axis=0)\n    # all_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))\n    return all_s\n\nif DEBUG and STEP_BY_STEP: \n    all_s = get_mean_predictions_1(model_list, features)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.696548Z","iopub.execute_input":"2024-10-18T13:17:23.696915Z","iopub.status.idle":"2024-10-18T13:17:23.709324Z","shell.execute_reply.started":"2024-10-18T13:17:23.696872Z","shell.execute_reply":"2024-10-18T13:17:23.708568Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2D CNN predictions","metadata":{}},{"cell_type":"markdown","source":"## input data preprocessing","metadata":{}},{"cell_type":"code","source":"def preprocess_2D(data_train, FGS_column):\n    print(\"FGS_column shape:\", FGS_column.shape)\n    print(\"data_train:\", data_train.shape)\n    dataset = np.concatenate([data_train, FGS_column[:,:, np.newaxis,:]], axis = 2)\n    # dataset = dataset.sum(axis=3)\n    dataset = np.nansum(dataset, axis=3)\n    print(\"dataset shape:\", dataset.shape)\n    np.isnan(dataset).sum()\n    return dataset\n\n\nif DEBUG and STEP_BY_STEP: \n    dataset = preprocess_2D(data_train, FGS_column)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.710352Z","iopub.execute_input":"2024-10-18T13:17:23.710644Z","iopub.status.idle":"2024-10-18T13:17:23.723203Z","shell.execute_reply.started":"2024-10-18T13:17:23.710613Z","shell.execute_reply":"2024-10-18T13:17:23.722357Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def norm_star_spectrum (signal):\n    img_star = signal[:, :50].mean(axis = 1) + signal[:, -50:].mean(axis = 1)\n    return signal / img_star[:, np.newaxis,:]\n\n\nif DEBUG and STEP_BY_STEP: \n    dataset_norm = norm_star_spectrum(dataset)\n    dataset_norm = np.transpose(dataset_norm,(0, 2, 1))\n    print(dataset_norm.shape)\n    input_data = dataset_norm.transpose(0, 2, 1)\n    print(input_data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.72427Z","iopub.execute_input":"2024-10-18T13:17:23.724857Z","iopub.status.idle":"2024-10-18T13:17:23.733493Z","shell.execute_reply.started":"2024-10-18T13:17:23.724815Z","shell.execute_reply":"2024-10-18T13:17:23.732713Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##### Substracting the out transit signal #####\ndef suppress_out_transit (data, ingress, egress) : \n    data_in = data[:, ingress:egress, :]\n    return data_in\n\n###### Substract the mean #####\ndef substract_data_mean(data):\n    data_mean = np.zeros(data.shape)\n    for i in range(data.shape[0]):\n        data_mean[i] = data[i] - data[i].mean()\n    return data_mean\n\n\nif DEBUG and STEP_BY_STEP: \n    ingress, egress = 75, 115\n    input_data = suppress_out_transit(input_data, ingress, egress)\n    input_data = substract_data_mean(input_data)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.734578Z","iopub.execute_input":"2024-10-18T13:17:23.734965Z","iopub.status.idle":"2024-10-18T13:17:23.743345Z","shell.execute_reply.started":"2024-10-18T13:17:23.734933Z","shell.execute_reply":"2024-10-18T13:17:23.742435Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DatasetNormalizer:\n    def __init__(self, predefined_limits):\n        self.min = predefined_limits[\"min\"]\n        self.max = predefined_limits[\"max\"]\n    \n    def normalize(self, df):\n        dataset_norm1 = np.zeros(df.shape)\n        dataset_norm1 = df / self.max \n        return dataset_norm1\n\nif DEBUG and STEP_BY_STEP: \n    train_limit_values = {\"min\": -0.028245810505925617, \"max\": 0.02620108573451435}\n    scaler = DatasetNormalizer(train_limit_values)\n    input_data_normed = scaler.normalize(input_data)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.744417Z","iopub.execute_input":"2024-10-18T13:17:23.744781Z","iopub.status.idle":"2024-10-18T13:17:23.755916Z","shell.execute_reply.started":"2024-10-18T13:17:23.74474Z","shell.execute_reply":"2024-10-18T13:17:23.755114Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## model inference","metadata":{}},{"cell_type":"code","source":"class CNN2DModel(pytorch_lightning.LightningModule):\n    def __init__(self, parameters):\n        super(CNN2DModel, self).__init__()\n\n        self.print = False\n\n        # tensor dim: batch, input_channels, time, wavelength\n        ############################################\n        # time convolutions\n        ############################################\n        self.conv_time = []\n        last_dim = 1\n        for i, _params in enumerate(parameters[\"layer1\"]):\n            # print(i, last_dim, _params)\n            self.conv_time.append(\n                nn.Conv2d(last_dim, _params[0], _params[1], padding='same')\n            )\n            last_dim = _params[0]\n\n        self.conv1 = self.conv_time[0]\n        self.conv2 = self.conv_time[1]\n        self.conv3 = self.conv_time[2]\n        self.conv4 = self.conv_time[3]\n\n        ############################################\n        # wavelength convolutions\n        ############################################\n        self.conv_wave = []\n        for i, _params in enumerate(parameters[\"layer2\"]):\n            # print(i, last_dim, _params)\n            self.conv_wave.append(\n                nn.Conv2d(last_dim, _params[0], _params[1], padding='same')\n            )\n            last_dim = _params[0]\n\n        self.conv5 = self.conv_wave[0]\n        self.conv6 = self.conv_wave[1]\n        self.conv7 = self.conv_wave[2]\n        self.conv8 = self.conv_wave[3]\n\n        ############################################\n        self.pool = nn.MaxPool2d((2, 1))\n        self.pool2 = nn.MaxPool2d((1, 2))\n        self.batch_norm1 = nn.BatchNorm2d(parameters[\"layer1\"][0][0])\n        self.batch_norm2 = nn.BatchNorm2d(parameters[\"layer2\"][0][0])\n\n        ############################################\n        # automatically sets the input size of the fc layers\n        x = self._conv_blocks(torch.rand((1, 1, 40, 283)), False)\n        n = x.shape[1] * x.shape[2] * x.shape[3]\n\n        self.flatten = nn.Flatten()\n        self.fc1 = nn.Linear(n, 700)  # Adjust dimensions as needed\n        self.dropout = nn.Dropout(0.2)\n        self.fc2 = nn.Linear(700, 283)\n\n        ############################################\n        self.criterion = nn.MSELoss()\n        self.train_mae = MeanAbsoluteError()\n        self.val_mae = MeanAbsoluteError()\n        self.best_val_loss = np.inf\n        self.best_val_mae = np.inf\n        \n    def _conv_blocks(self, x, verbose):\n        \n        if verbose: print(\"-input-\", x.shape)\n        if verbose: print(\"############### starts convolution on 'time'\")\n        # ----------------------------------------\n        # conv 1 block \n        x = F.relu(self.conv1(x))\n        x = self.pool(x) \n        x = self.batch_norm1(x)\n        if verbose: print(\"-conv1 block-\", x.shape) \n\n        # ----------------------------------------\n        # conv 2 block\n        x = F.relu(self.conv2(x))\n        x = self.pool(x)\n        if verbose: print(\"-conv2 block-\", x.shape)\n\n        # ----------------------------------------\n        # conv 3 block\n        x = F.relu(self.conv3(x))\n        x = self.pool(x)\n        if verbose: print(\"-conv3 block-\", x.shape)\n\n        # ----------------------------------------\n        # conv 4 block\n        x = F.relu(self.conv4(x))\n        if verbose: print(\"-conv4 block-\", x.shape)\n\n        if verbose: print(\"############### starts convolution on 'wavelengths'\")\n        # ----------------------------------------\n        # conv 5 block\n        x = F.relu(self.conv5(x))\n        x = self.pool2(x)\n        if verbose: print(\"-conv5 block-\", x.shape)\n        x = self.batch_norm2(x)\n        \n        # ----------------------------------------\n        # conv 6 block\n        x = F.relu(self.conv6(x))\n        x = self.pool2(x)\n        if verbose: print(\"-conv6 block-\", x.shape)\n\n         # ----------------------------------------\n        # conv 7 block\n        x = F.relu(self.conv7(x))\n        x = self.pool2(x)\n        if verbose: print(\"-conv7 block-\", x.shape)\n\n        # ----------------------------------------\n        # conv 8 block\n        x = F.relu(self.conv8(x))\n        x = self.pool2(x)\n        if verbose: print(\"-conv8 block-\", x.shape)\n\n        return x\n\n    def forward(self, x):\n\n        x = self._conv_blocks(x, self.print)\n        \n        # -----------------------------------------\n        if self.print: print(\"############### starts fully connected layers\")\n        x = self.flatten(x)\n        if self.print: print(\"-flattened-\", x.shape)\n        \n        x = F.relu(self.fc1(x))\n        if self.print: print(\"-fc1-\", x.shape)\n        x = self.dropout(x)\n\n        x = self.fc2(x)\n        if self.print: print(\"-fc2-\", x.shape)\n        \n        return x\n\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        # print(y_hat.shape, y.shape)\n        loss = self.criterion(y_hat, y)\n        self.train_mae(y_hat, y)\n        self.log('train_loss', loss, prog_bar=True, on_epoch=False)\n        self.log('train_mae', self.train_mae, prog_bar=True, on_epoch=False)\n        return loss\n\n    def validation_step(self, batch, batch_idx, on_epoch=True):\n        x, y = batch\n        y_hat = self(x)\n        loss = self.criterion(y_hat, y)\n        self.val_mae(y_hat, y)\n        self.log('val_loss', loss, prog_bar=True)\n        self.log('val_mae', self.val_mae, prog_bar=True)\n        \n        self.best_val_loss = min(loss, self.best_val_loss)\n        # print(\"\\n\", loss.item(), self.best_val_loss.item())\n        self.best_val_mae = min(self.val_mae, self.best_val_mae)\n        self.log('best_val_loss', self.best_val_loss, prog_bar=False)\n        self.log('best_val_mae', self.best_val_mae, prog_bar=False)\n        \n        return loss\n\n    def configure_optimizers(self):\n        optimizer = Adam(self.parameters(), lr=0.001)\n        # optimizer = torch.optim.SGD(self.parameters(), lr=0.001)\n    \n        sch  = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer,\n            mode='min',\n            factor=0.2,\n            patience=10, # adjust as you want\n            verbose=True,\n            cooldown=5,\n            min_lr=1e-5,\n        )\n        return {\n            'optimizer': optimizer,\n            'lr_scheduler': sch, \n            'monitor': 'val_loss'}\n       \n","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.757264Z","iopub.execute_input":"2024-10-18T13:17:23.75761Z","iopub.status.idle":"2024-10-18T13:17:23.786288Z","shell.execute_reply.started":"2024-10-18T13:17:23.75757Z","shell.execute_reply":"2024-10-18T13:17:23.78536Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if DEBUG and STEP_BY_STEP: \n#     models = [\n#         (\"model7_params_1_8d3917a28.pkl\", \"model_2dcnn_1_8d3917a28-epoch057-val_loss0.00021.ckpt\")\n#     ]\n    \n#     for model_param, model_ckpt in models:\n#         parameters = pickle.load(open(f\"../input/adc24-checkpoints/{model_param}\", \"rb\"))\n#         model = CNN2DModel.load_from_checkpoint(f\"../input/adc24-checkpoints/{model_ckpt}\", parameters=parameters)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.787524Z","iopub.execute_input":"2024-10-18T13:17:23.78809Z","iopub.status.idle":"2024-10-18T13:17:23.801371Z","shell.execute_reply.started":"2024-10-18T13:17:23.788047Z","shell.execute_reply":"2024-10-18T13:17:23.80049Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if DEBUG and STEP_BY_STEP: \n    targets_norm_dict = pd.read_pickle(\"../input/adc24-checkpoints/target_norms.pkl\")\n    targets_abs_max = targets_norm_dict[\"targets_abs_max\"]","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.802419Z","iopub.execute_input":"2024-10-18T13:17:23.80279Z","iopub.status.idle":"2024-10-18T13:17:23.811318Z","shell.execute_reply.started":"2024-10-18T13:17:23.802748Z","shell.execute_reply":"2024-10-18T13:17:23.810544Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def targets_norm_back (data, data_abs_max) : \n    return data * data_abs_max\n\n\ndef NN_uncertainity(model, x_test, targets_abs_max, T=5):\n    predictions = []\n    for _ in range(T):\n        pred_norm = model.to(\"cpu\").double()(torch.tensor(x).to(\"cpu\"))\n        pred = targets_norm_back(pred_norm, targets_abs_max)\n        predictions += [pred.detach().numpy()]  \n        # print(pred.shape)\n    \n    mean, std = np.mean(np.array(predictions), axis=0), np.std(np.array(predictions), axis=0)\n    return mean, std\n\n\ndef get_predictions_2D(input_data_normed, targets_abs_max, model):\n    \n    nb_dropout = 5\n    do_the_mcdropout = False\n    \n    if do_the_mcdropout :\n        x = valid_obs_norm[:, np.newaxis, :, :].astype(np.float64)\n        spectre_valid_shift, spectre_valid_shift_std = NN_uncertainity(model.double(), x, targets_abs_max, T = nb_dropout)\n\n    else :\n        x = input_data_normed[:, np.newaxis, :, :].astype(np.float64)\n        pred_valid_norm = model.cpu().double()(torch.tensor(x))\n        pred_valid = targets_norm_back(pred_valid_norm, targets_abs_max)\n        spectre_valid_shift = pred_valid\n        spectre_valid_shift_std = spectre_valid_shift*0.1\n\n    spectre_valid_shift = spectre_valid_shift.detach().cpu().numpy()\n    spectre_valid_shift_std = spectre_valid_shift_std.detach().cpu().numpy()\n    return spectre_valid_shift, spectre_valid_shift_std\n\ndef get_fluctuations_predictions(input_data_normed, targets_abs_max, models):\n    spectra_mean, spectra_std = [], []\n    for model_param, model_ckpt in models:\n        parameters = pickle.load(open(f\"../input/adc24-checkpoints/{model_param}\", \"rb\"))\n        model = CNN2DModel.load_from_checkpoint(f\"../input/adc24-checkpoints/{model_ckpt}\", parameters=parameters)\n        \n        spectre_valid_shift, spectre_valid_shift_std = get_predictions_2D(input_data_normed, targets_abs_max, model)\n        spectra_mean.append(spectre_valid_shift)\n        spectra_std.append(spectre_valid_shift_std)\n    \n    spectre_valid_shift = np.array(spectra_mean).mean(axis=0)\n    spectre_valid_std = np.array(spectra_std).mean(axis=0)\n    \n    return spectre_valid_shift, spectre_valid_std\n\n\nif DEBUG and STEP_BY_STEP: \n\n    models = [\n        # (\"model7_params_1_8d3917a28.pkl\", \"model_2dcnn_1_8d3917a28-epoch057-val_loss0.00021.ckpt\"),\n        # (\"model7_params_3_66265fbb6.pkl\", \"model_2dcnn_3_66265fbb6-epoch042-val_loss0.00022.ckpt\"),\n        # (\"model7_params_1_66265fbb6.pkl\", \"model_2dcnn_1_1_66265fbb6-epoch071-val_loss0.00008.ckpt\"),\n        (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_0_b98fe5a67-epoch=032-val_loss=0.00025.ckpt\")\n    ]\n    \n    spectre_valid_shift, spectre_valid_shift_std = get_fluctuations_predictions(input_data_normed, targets_abs_max, models)\n       ","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.812535Z","iopub.execute_input":"2024-10-18T13:17:23.813564Z","iopub.status.idle":"2024-10-18T13:17:23.827148Z","shell.execute_reply.started":"2024-10-18T13:17:23.813522Z","shell.execute_reply":"2024-10-18T13:17:23.8263Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if DEBUG:\n#     plt.figure()\n#     for i in range (5) : \n#         plt.plot(spectre_valid_shift[-i]+0.0001*i, '-', alpha = 0.5)\n#     plt.title('Spectra predictions for the validation set') \n#     plt.xlabel('Time')\n#     plt.ylabel('Arbitrary flux')\n#     plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.828175Z","iopub.execute_input":"2024-10-18T13:17:23.82847Z","iopub.status.idle":"2024-10-18T13:17:23.841416Z","shell.execute_reply.started":"2024-10-18T13:17:23.828439Z","shell.execute_reply":"2024-10-18T13:17:23.840491Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Combine 1D and 2D CNN output for FINAL SPECTRA ","metadata":{}},{"cell_type":"code","source":"######## ADD THE FLUCTUATIONS TO THE MEAN ########\ndef add_the_mean (shift, mean) : \n    return shift + mean[:, np.newaxis]\n\nif DEBUG and STEP_BY_STEP: \n    predictions_valid = add_the_mean(spectre_valid_shift, all_s)\n    predictions_std_valid = spectre_valid_shift_std**2\n","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.842489Z","iopub.execute_input":"2024-10-18T13:17:23.842823Z","iopub.status.idle":"2024-10-18T13:17:23.850926Z","shell.execute_reply.started":"2024-10-18T13:17:23.842792Z","shell.execute_reply":"2024-10-18T13:17:23.850198Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if DEBUG and STEP_BY_STEP: \n    solution = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_labels.csv', index_col=0)\n    per_planet_mean = solution.mean(axis=1).values\n    normalized_targets = solution.values / per_planet_mean[:, np.newaxis]\n    target_mean_value = normalized_targets.mean(axis=0)\n    target_std_value = normalized_targets.std(axis=0) \n\n    targets = solution.loc[adc_info.index]\n    uncertainty = predictions_std_valid\n\n    wavelength = np.loadtxt(f'{path_folder}/wavelengths.csv', skiprows=1, delimiter = ',')\n\n\n    def plot_one_sample_valid(ax, p):\n        ax.set_title(f'Result for sample {p} ')\n        line1, = ax.plot(wavelength, predictions_valid[p], 'tomato', linewidth=\"2\", label='Prediction')\n        line2, = ax.plot(wavelength, targets.values[p], '.k', label='Target')\n        line3 = ax.axhline(all_s[p], linestyle=\"--\", linewidth=0.5, c='tomato', label='mean Target prediction')\n        ax.fill_between(wavelength, predictions_valid[p, :] - uncertainty[p], predictions_valid[p, :] + uncertainty[p], color='silver', alpha=0.8, label='Uncertainty')\n        ax.set_ylabel(f'$(R_p/R_s)^2$')\n        ax.set_xlabel(f'Wavelength ($\\mu$m)')\n        ax.grid()\n        return line1, line2, line3\n\n\n    num_samples = 5\n    rows, cols = 3, 2\n\n    fig, axs = plt.subplots(rows, cols, figsize=(12, 8))\n    samples = [0, 1, 2, 3, 4, 5]\n    lines = []\n\n    for i, ax in enumerate(axs.flat):\n        if i >= len(samples): break\n        lines.extend(plot_one_sample_valid(ax, samples[i]))\n\n    fig.legend(lines[:2], ['Prediction', 'Target'], loc='upper center', ncol=3, bbox_to_anchor=(0.5, -0.05))\n    fig.suptitle('Validation dataset')\n    plt.tight_layout()\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.85197Z","iopub.execute_input":"2024-10-18T13:17:23.852257Z","iopub.status.idle":"2024-10-18T13:17:23.864859Z","shell.execute_reply.started":"2024-10-18T13:17:23.852228Z","shell.execute_reply":"2024-10-18T13:17:23.864111Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create submission file","metadata":{}},{"cell_type":"code","source":"def postprocessing(pred_array, index, sigma_pred):\n    \"\"\"Create a submission dataframe from its components\n    \n    Parameters:\n    pred_array: ndarray of shape (n_samples, 283)\n    index: pandas.Index of length n_samples with name 'planet_id'\n    sigma_pred: float\n    \n    Return value:\n    df: DataFrame of shape (n_samples, 566) with planet_id as index\n    \"\"\"\n    wavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')\n    df = pd.concat(\n        [pd.DataFrame(pred_array.clip(0, None), index=index, columns=wavelengths.columns),\n         pd.DataFrame(sigma_pred, index=index, columns=[f\"sigma_{i}\" for i in range(1, 284)])],\n         axis=1\n    )\n    df.index.name = \"planet_id\"\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.865851Z","iopub.execute_input":"2024-10-18T13:17:23.866148Z","iopub.status.idle":"2024-10-18T13:17:23.879148Z","shell.execute_reply.started":"2024-10-18T13:17:23.866118Z","shell.execute_reply":"2024-10-18T13:17:23.878307Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_submission_file(predictions_valid, predictions_std_valid, index, submission_file_index): \n    wavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')\n    submit = postprocessing(predictions_valid, index, predictions_std_valid)\n    display(submit.sample(min(3, submit.shape[0])))\n    submit.to_csv(f'submission_{submission_file_index}.csv')\n\nif STEP_BY_STEP: \n    index = adc_info.index\n    create_submission_file(predictions_valid, predictions_std_valid, index, 0)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.880468Z","iopub.execute_input":"2024-10-18T13:17:23.880979Z","iopub.status.idle":"2024-10-18T13:17:23.893391Z","shell.execute_reply.started":"2024-10-18T13:17:23.880938Z","shell.execute_reply":"2024-10-18T13:17:23.892604Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if STEP_BY_STEP: \n    sigma = np.ones(shape=(all_s.shape[0], 283))\n    sigma = sigma * 1.25e-4\n    create_submission_file(predictions_valid , sigma, adc_info.index, 0)\n    submit = pd.read_csv(f'submission_{0}.csv', index_col=0)\n\n    if DEBUG:\n        test_sub_file_name = \"sub_file_emulation.csv\"\n    else:\n        test_sub_file_name = \"submission.csv\"\n\n    submit.to_csv(test_sub_file_name)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.894586Z","iopub.execute_input":"2024-10-18T13:17:23.895365Z","iopub.status.idle":"2024-10-18T13:17:23.904413Z","shell.execute_reply.started":"2024-10-18T13:17:23.895322Z","shell.execute_reply":"2024-10-18T13:17:23.90363Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_pipeline(adc_info, submission_file_index):\n    data_train, FGS_column = preprocess_data(adc_info)\n    index = adc_info.index\n    \n    pre_train = get_pre_train_data(data_train, FGS_column)\n\n    model_list = [\n        (\"features_model8_2.pkl\", \"reg_model8_fold<NUMBER>_2.pkl\"),\n        (\"features_model8_3.pkl\", \"reg_model8_fold<NUMBER>_3.pkl\"),\n        (\"features_model8_4.pkl\", \"reg_model8_fold<NUMBER>_4.pkl\"),\n        (\"features_model8_5.pkl\", \"reg_model8_fold<NUMBER>_5.pkl\"),\n        (\"features_model8_6.pkl\", \"reg_model8_fold<NUMBER>_6.pkl\"),\n        (\"features_model8_7.pkl\", \"reg_model8_fold<NUMBER>_7.pkl\"),\n        (\"features_model8_8.pkl\", \"reg_model8_fold<NUMBER>_8.pkl\"),\n    ]\n    features = get_fluxes(model_list, adc_info, pre_train)\n    all_s = get_mean_predictions_1(model_list, features)\n    \n    dataset = preprocess_2D(data_train, FGS_column)\n    dataset_norm = norm_star_spectrum(dataset)\n    dataset_norm = np.transpose(dataset_norm,(0, 2, 1))\n    input_data = dataset_norm.transpose(0, 2, 1)    ############# ????????????????????\n    \n\n    ingress, egress = 75, 115\n    input_data = suppress_out_transit(input_data, ingress, egress)\n    input_data = substract_data_mean(input_data)\n\n\n    train_limit_values = {\"min\": -0.028245810505925617, \"max\": 0.02620108573451435}\n\n    scaler = DatasetNormalizer(train_limit_values)\n    input_data_normed = scaler.normalize(input_data)\n    \n    targets_norm_dict = pd.read_pickle(\"../input/adc24-checkpoints/target_norms.pkl\")\n    targets_abs_max = targets_norm_dict[\"targets_abs_max\"]\n\n    models = [\n        (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_0_b98fe5a67-epoch032-val_loss0.00025.ckpt\"),\n        (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_0_b98fe5a67-epoch043-val_loss0.00026.ckpt\"),\n        (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_0_b98fe5a67-epoch048-val_loss0.00027.ckpt\"),\n        # (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_1_b98fe5a67-epoch053-val_loss0.00007.ckpt\"),\n        # (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_1_b98fe5a67-epoch063-val_loss0.00007.ckpt\"),\n        # (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_1_b98fe5a67-epoch082-val_loss0.00007.ckpt\"),\n        (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_2_b98fe5a67-epoch037-val_loss0.00010.ckpt\"),\n        (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_2_b98fe5a67-epoch047-val_loss0.00010.ckpt\"),\n        (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_2_b98fe5a67-epoch048-val_loss0.00010.ckpt\"),\n        (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_3_b98fe5a67-epoch039-val_loss0.00010.ckpt\"),\n        (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_3_b98fe5a67-epoch044-val_loss0.00009.ckpt\"),\n        (\"model10_params_1_b98fe5a67.pkl\", \"model10_2dcnn_1_3_b98fe5a67-epoch047-val_loss0.00010.ckpt\"),\n    ]\n    \n    spectre_valid_shift, spectre_valid_shift_std = get_fluctuations_predictions(input_data_normed, targets_abs_max, models)\n    \n    predictions_valid = add_the_mean(spectre_valid_shift, all_s)\n\n    predictions_std_valid = spectre_valid_shift_std**2\n\n    sigma = np.ones(shape=(all_s.shape[0], 283))\n    sigma = sigma * 1.4e-4\n    \n    create_submission_file(predictions_valid, sigma, index, submission_file_index)\n ","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.905835Z","iopub.execute_input":"2024-10-18T13:17:23.90619Z","iopub.status.idle":"2024-10-18T13:17:23.918032Z","shell.execute_reply.started":"2024-10-18T13:17:23.906149Z","shell.execute_reply":"2024-10-18T13:17:23.917172Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not STEP_BY_STEP:\n    nb = adc_info.shape[0]\n    print(\"nb of exoplanets\", nb)\n\n    step = 150\n    nb_iter = nb // step + 1\n    for i in range(nb_iter):\n        print(90*\"#\")\n        print(f\"step n°{i}\")\n        print(90*\"#\")\n        run_pipeline(adc_info[i * step : (i+1) * step], submission_file_index = i)\n\n    if DEBUG:\n        test_sub_file_name = \"sub_file_emulation.csv\"\n    else:\n        test_sub_file_name = \"submission.csv\"\n\n    submission_files = []\n    for i in range(nb_iter):\n        submission_files.append(pd.read_csv(f'submission_{i}.csv', index_col=0))\n    submit = pd.concat(submission_files)\n    # submit.to_csv(test_sub_file_name)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T13:17:23.919321Z","iopub.execute_input":"2024-10-18T13:17:23.919739Z","iopub.status.idle":"2024-10-18T13:18:54.470017Z","shell.execute_reply.started":"2024-10-18T13:17:23.919697Z","shell.execute_reply":"2024-10-18T13:18:54.469011Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit = submit.join(adc_info[[\"star\"]])\n\nstar0 = submit[submit[\"star\"] == 0]\nstar1 = submit[submit[\"star\"] == 1]\nstar2 = submit[~submit[\"star\"].isin([0, 1])]\n\nfactor = {0: 1.05, 1: 0.5, 2: 1.05}\nnew_sub = []\nfor i, star in enumerate([star0, star1, star2]):\n    a = star[[f'wl_{c}' for c in range(1, 284)]] \n    b = star[[f'sigma_{c}' for c in range(1, 284)]] * factor[i]\n    new_sub.append(pd.concat([a, b], axis=1))\nnew_sub = pd.concat(new_sub)  \nnew_sub = new_sub.loc[adc_info.index]\nnew_sub.to_csv(test_sub_file_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T13:18:54.475524Z","iopub.execute_input":"2024-10-18T13:18:54.476022Z","iopub.status.idle":"2024-10-18T13:18:54.511952Z","shell.execute_reply.started":"2024-10-18T13:18:54.475976Z","shell.execute_reply":"2024-10-18T13:18:54.511044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if DEBUG:\n    preds = pd.read_csv(\"sub_file_emulation.csv\", index_col=0)\n    targets = pd.read_csv(f'{path_folder}/train_labels.csv', index_col=0)\n    overall_score = np.mean((preds[targets.columns].values - targets.loc[preds.index].values)**2)**0.5 * 1E+6\n    print(f\"RMSE :  {overall_score:9.3f} ppm\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T13:18:54.513302Z","iopub.execute_input":"2024-10-18T13:18:54.513657Z","iopub.status.idle":"2024-10-18T13:18:54.655937Z","shell.execute_reply.started":"2024-10-18T13:18:54.513618Z","shell.execute_reply":"2024-10-18T13:18:54.654957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if DEBUG:\n#     mean_flux = preds[[c for c in preds.columns if \"sigma\" not in c]].mean(axis=1)\n#     uncertainty = preds[[c for c in preds.columns if \"sigma\" in c]].mean(axis=1)\n#     plt.scatter(mean_flux.values, uncertainty.values, marker = \"o\", s=8)\n#     plt.title(\"uncertainty\")\n#     plt.xlabel(\"mean flux\")\n#     plt.ylabel(\"std\")\n#     plt.grid()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T13:18:54.657272Z","iopub.execute_input":"2024-10-18T13:18:54.657584Z","iopub.status.idle":"2024-10-18T13:18:54.664226Z","shell.execute_reply.started":"2024-10-18T13:18:54.657551Z","shell.execute_reply":"2024-10-18T13:18:54.663198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}