{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":70367,"databundleVersionId":9188054},{"sourceType":"datasetVersion","sourceId":9651823,"datasetId":5613665,"databundleVersionId":9879033},{"sourceType":"datasetVersion","sourceId":9433826,"datasetId":5731720,"databundleVersionId":9638969}],"dockerImageVersionId":30762,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n# import itertools\nimport os\nimport glob \nimport random\nimport polars as pl\nimport gc\nimport pickle\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pytorch_lightning \nimport time\nfrom torchmetrics import MeanAbsoluteError\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport cupy as cp    \n\nfrom scipy.optimize import minimize\nfrom scipy import optimize\n\ntorch.manual_seed(0)\nrandom.seed(0)\nnp.random.seed(0)\nt0 = time.time()\n\nprint(np.__version__)\nprint(torch.__version__)\nprint(pytorch_lightning.__version__)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-18T07:09:16.411177Z","iopub.execute_input":"2024-10-18T07:09:16.411528Z","iopub.status.idle":"2024-10-18T07:09:16.422529Z","shell.execute_reply.started":"2024-10-18T07:09:16.411491Z","shell.execute_reply":"2024-10-18T07:09:16.421446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install -U scikit-learn==1.5.2 -f ../input/sklearn1-5-2 --no-index","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:08:59.896777Z","iopub.execute_input":"2024-10-18T07:08:59.897597Z","iopub.status.idle":"2024-10-18T07:09:16.409063Z","shell.execute_reply.started":"2024-10-18T07:08:59.897554Z","shell.execute_reply":"2024-10-18T07:09:16.408011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# UNCERTAINTY_SCALING_FACTOR = 6\nDEBUG = False","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:09:16.423622Z","iopub.execute_input":"2024-10-18T07:09:16.423913Z","iopub.status.idle":"2024-10-18T07:09:16.434253Z","shell.execute_reply.started":"2024-10-18T07:09:16.423883Z","shell.execute_reply":"2024-10-18T07:09:16.433342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def create_fake_submission_file():\n#     wavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')\n#     fake_submit = pd.DataFrame(index = index, columns=list(wavelengths.columns) + [c.replace(\"wl_\", \"sigma_\") for c in wavelengths.columns])\n#     fake_submit.fillna(0, inplace=True)\n#     fake_submit.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:09:16.619589Z","iopub.execute_input":"2024-10-18T07:09:16.619927Z","iopub.status.idle":"2024-10-18T07:09:16.62398Z","shell.execute_reply.started":"2024-10-18T07:09:16.619894Z","shell.execute_reply":"2024-10-18T07:09:16.622989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_folder = '/kaggle/input/ariel-data-challenge-2024/' # path to the folder containing the data\npath_out = './preprocessed_data/' # path to the folder to store the light data\noutput_dir = './preprocessed_data/' # path for the output directory\n\nif not os.path.exists(path_out):\n    os.makedirs(path_out)\n    print(f\"Directory {path_out} created.\")\nelse:\n    print(f\"Directory {path_out} already exists.\")","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:09:17.552002Z","iopub.execute_input":"2024-10-18T07:09:17.552382Z","iopub.status.idle":"2024-10-18T07:09:17.558313Z","shell.execute_reply.started":"2024-10-18T07:09:17.552344Z","shell.execute_reply":"2024-10-18T07:09:17.557472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pre-processing the data","metadata":{}},{"cell_type":"code","source":"def ADC_convert(signal, gain, offset):\n    signal /= gain\n    signal += offset\n    return signal\n\ndef clean_flat_dark_dead(signal, dead, flat, dark):\n    signal = (signal - dark ) / (flat - dark)\n    dead_mask = cp.asarray((dead == 1.0).reshape((-1,)))\n    signal[:, dead_mask] = cp.nan\n    return signal\n\ndef apply_linear_corr(linear_corr, signal):\n    result = linear_corr[5, :]\n    for i in range(4, -1, -1):\n        result = result * signal + linear_corr[i, :]\n    return result\n\n\ndef bin_obs(signal, binning):\n    signal_binned = cp.zeros((signal.shape[0] // binning, signal.shape[1], signal.shape[2]))\n    for i in range(signal.shape[0] // binning):\n        signal_binned[i, :, :] = cp.sum(signal[i * binning : (i + 1) * binning, :, :], axis=0)\n    return signal_binned\n\n\ndef load_signal_data(planet_id, dataset, instrument, adc_info, step_size=30):\n    precision = pl.Float32 # pl.Float64\n    binning = step_size if instrument == \"AIRS-CH0\" else step_size * 12  # 4642\n    img_size = (32, 356) if instrument == \"AIRS-CH0\" else (32, 32)\n    gain, offset = (\n        adc_info.loc[planet_id][f\"{instrument}_adc_gain\"],\n        adc_info.loc[planet_id][f\"{instrument}_adc_offset\"],\n    )\n    signal_file = f\"{path_folder}/{dataset}/{planet_id}/{instrument}_signal.parquet\"\n    calibration_file = f\"{path_folder}/{dataset}/{planet_id}/{instrument}_calibration\"\n    \n    signal = cp.array(pl.read_parquet(signal_file).cast(precision).to_numpy())\n\n    flat = cp.array(pl.read_parquet(f\"{calibration_file}/flat.parquet\").cast(precision).to_numpy().reshape((1, -1)))\n    dark = cp.array(pl.read_parquet(f\"{calibration_file}/dark.parquet\").cast(precision).to_numpy().reshape((1, -1)))\n    dead = cp.array(pl.read_parquet(f\"{calibration_file}/dead.parquet\").cast(precision).to_numpy().reshape((1, -1)))\n  \n    linear_corr = cp.array(\n        pl.read_parquet(f\"{calibration_file}/linear_corr.parquet\").cast(precision).to_numpy().reshape(6, -1)\n    )\n\n    signal = ADC_convert(signal, gain, offset)  # AIRS: dim=(11'250, 11'392)\n    signal = clean_flat_dark_dead(signal, dead, flat, dark)\n    signal = apply_linear_corr(linear_corr, signal) \n    signal = signal.reshape((signal.shape[0], img_size[0], img_size[1]))\n    signal = signal[1::2] - signal[0::2]    \n    signal = bin_obs(signal, binning)\n    \n    if instrument == \"AIRS-CH0\":\n        return cp.asnumpy(signal)[:, :, 39:321]\n    else:\n        return cp.asnumpy(signal)\n    \ndef generate_signal(mode, adc_info, instrument=\"FGS1\",):\n    assert instrument in [\"FGS1\", \"AIRS-CH0\"]\n    signal = []\n    for planet_id in tqdm(adc_info.index):\n        signal.append(load_signal_data(planet_id, mode, instrument, adc_info))\n    return np.stack(signal).transpose((0, 1, 3, 2))","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:09:19.266383Z","iopub.execute_input":"2024-10-18T07:09:19.267055Z","iopub.status.idle":"2024-10-18T07:09:19.286464Z","shell.execute_reply.started":"2024-10-18T07:09:19.267015Z","shell.execute_reply":"2024-10-18T07:09:19.285542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    files = glob.glob(os.path.join(path_folder + 'train/', '*/*'))\n    files = files[:150]  # 65 -> 16 exoplanets ; 150 -> 38 exoplanets \n#     files = files[:25]\n    \n    adc_info = pd.read_csv(os.path.join(path_folder, 'train_adc_info.csv'))\n    adc_info.set_index(\"planet_id\", inplace=True)\n    \n    exoplanet_ids = list(set([int(c.split(\"/\")[-2]) for c in files]))\n    adc_info = adc_info[adc_info.index.isin(exoplanet_ids)]\n    print(exoplanet_ids)\n\nelse:\n    files = glob.glob(os.path.join(path_folder + 'test/', '*/*'))\n    \n    adc_info = pd.read_csv(os.path.join(path_folder, 'test_adc_info.csv'))\n    adc_info.set_index(\"planet_id\", inplace=True)\n\n# index = adc_info.index\n\nprint(len(files))\nprint(\"adc_info's shape:\", adc_info.shape)\nt1 = time.time()\n\nN = len(adc_info.index.unique())\n\nif DEBUG: print(f\"[{t1-t0:6.1f}s]  {N} exoplanets\")","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:09:21.744568Z","iopub.execute_input":"2024-10-18T07:09:21.745574Z","iopub.status.idle":"2024-10-18T07:09:23.41801Z","shell.execute_reply.started":"2024-10-18T07:09:21.74552Z","shell.execute_reply":"2024-10-18T07:09:23.41705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_data(adc_info):\n    field = (\"test\", \"train\")[DEBUG]\n    data_train_FGS = generate_signal(field, adc_info, instrument=\"FGS1\")\n    data_train = generate_signal(field, adc_info, instrument=\"AIRS-CH0\")\n    print(\"shapes:\", data_train_FGS.shape, data_train.shape)\n    FGS_column = np.nanmean(data_train_FGS, axis = 2)\n    return data_train, FGS_column\n\n\ndata_train, FGS_column = preprocess_data(adc_info)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:09:23.419553Z","iopub.execute_input":"2024-10-18T07:09:23.419884Z","iopub.status.idle":"2024-10-18T07:09:44.618549Z","shell.execute_reply.started":"2024-10-18T07:09:23.419849Z","shell.execute_reply":"2024-10-18T07:09:44.617662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions","metadata":{}},{"cell_type":"code","source":"fgs_summed = np.nanmean(FGS_column, axis = 2)\nairs_summed = np.nanmean(data_train[:, :, :, 10:22], axis = 3)\npre_train = np.concatenate([fgs_summed[:, :, np.newaxis], airs_summed], axis=2)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:09:48.440813Z","iopub.execute_input":"2024-10-18T07:09:48.44121Z","iopub.status.idle":"2024-10-18T07:09:48.474157Z","shell.execute_reply.started":"2024-10-18T07:09:48.441173Z","shell.execute_reply":"2024-10-18T07:09:48.473161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SHOW_PLOT = False\n\ndef phase_detector(signal):\n    \n    MIN = np.argmin(signal[30:140])+30\n    signal1 = signal[:MIN ]\n    signal2 = signal[MIN :]\n\n    first_derivative1 = np.gradient(signal1)\n    first_derivative1 /= first_derivative1.max()\n    first_derivative2 = np.gradient(signal2)\n    first_derivative2 /= first_derivative2.max()\n\n    phase1 = np.argmin(first_derivative1)\n    phase2 = np.argmax(first_derivative2) + MIN\n\n    return phase1, phase2\n  \n\ndef best_ratio_search(signal, p1, p2, poylomial_order, delta=3):\n    DELTA = delta\n    def objective(s, signal):\n        \"\"\" on applique un facteur multiplicatif à la zone de transit => courbe y.\n            on cherche le facteur qui minimise l'écart entre le courbe 'scalée' y et un fit polynomial d'ordre 3.\n        \"\"\"\n        best_q = 1e10\n        \n        x = list(range(signal.shape[0]-DELTA*4))\n\n        length = signal.shape[0]\n        signal = np.array(list(zip(range(length), signal.tolist())))\n        signal = pd.DataFrame(signal[:, 1], index=signal[:, 0])\n        tmp = pd.concat([signal.iloc[:p1 - DELTA], signal.iloc[p1 + DELTA:p2-DELTA] * (1+s), signal.iloc[p2+DELTA:]])\n        x = list(tmp.index)\n        y = list(tmp[0].values)            \n        z = np.polyfit(x, y, deg=poylomial_order)\n        p = np.poly1d(z)\n        q = np.abs(p(x) - y).mean()\n        \n        if q < best_q :\n            best_q = q\n        \n        return q\n    \n    r = minimize(\n                objective,\n                [0.0001],   # première valeur de s ===> paramètre x\n                method= 'Nelder-Mead',\n                args = (signal)\n                  )\n    s = r.x[0]\n    return s\n\ntransit_limiting_timestamps = []\ndef get_ratio(features, pre_train):\n    \n    all_s = []\n    partial_s = []\n    for i in tqdm(range(pre_train.shape[0])):\n\n        # on prend une exoplenete et on fait la moyenne sur les longueur d'onde\n        # FGS est supprimé pour déterminer la période de transit\n        signal = pre_train[i,:,1:].mean(axis=1)   \n        # détermination des débuts / fin de transit\n        p1, p2 = phase_detector(signal)  \n        transit_limiting_timestamps.append([p1, p2])\n        \n        ###############################\n        # TO DEBUG\n        if SHOW_PLOT:\n            print(p1, p2)\n            plt.plot(signal) ; \n            plt.axvline(p1, color='r') ; plt.axvline(p2, color='r') ; \n            plt.grid()\n        ###############################\n        \n        # field=f'{poylomial_order}_{k}_{delta}_{NWAVE}'\n        ratios = {}\n        agg_features = []\n        for feature in features:\n            if feature.startswith(\"tot\") : \n                agg_features.append(feature)\n                continue\n                \n            poylomial_order, k, delta, NWAVE = map(int, feature.split(\"_\"))     \n            signal = pre_train[i,:,k-NWAVE:k+NWAVE].mean(axis=1)\n            field=f'{poylomial_order}_{k}_{delta}_{NWAVE}'\n            s = best_ratio_search(signal, p1, p2, poylomial_order, delta=delta)\n            ratios[field]= s \n        partial_s.append(ratios)\n        \n        ratios = {}\n        for feature in agg_features:\n            _, delta, order = feature.split(\"_\")\n            delta = int(delta.strip(\"delta=\"))\n            order = int(order.strip(\"order=\"))\n            # ratio avec sommation sur l'ensemble des longueurs d'onde\n            signal = pre_train[i,:,1:].mean(axis=1)\n            s = best_ratio_search(signal, p1, p2, poylomial_order, delta=delta)\n            ratios[feature] = s\n        all_s.append(ratios)\n            \n        # print(s, np.mean(ratios))\n        \n        ###############################\n        # TO DEBUG\n        if SHOW_PLOT:\n            DELTA = 3\n            print(\"ratio:\", s)\n            length = signal.shape[0]\n            signal = np.array(list(zip(range(length), signal.tolist())))\n            signal = pd.DataFrame(signal[:, 1], index=signal[:, 0])\n            tmp = pd.concat([signal.iloc[:p1 - DELTA], signal.iloc[p1 + DELTA:p2-DELTA] * (1+s), signal.iloc[p2+DELTA:]])\n            x = list(tmp.index)\n            y = list(tmp[0].values)\n            z = np.polyfit(x, y, deg=3)\n            p = np.poly1d(z)\n            plt.plot(x, p(x))\n            plt.plot(tmp.loc[p1:p2].index.values, tmp.loc[p1:p2][0].values)\n            plt.title(f\"target = {target_mean[i]:8.5f} / pred = {s:8.5f} / ratio = {s / target_mean[i] : 7.3f}\")\n            plt.show()\n        ###############################\n    \n        \n    # all_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))        \n    return all_s, partial_s, transit_limiting_timestamps\n","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:09:49.84385Z","iopub.execute_input":"2024-10-18T07:09:49.844732Z","iopub.status.idle":"2024-10-18T07:09:49.868692Z","shell.execute_reply.started":"2024-10-18T07:09:49.844689Z","shell.execute_reply":"2024-10-18T07:09:49.867727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## model mean flux","metadata":{}},{"cell_type":"code","source":"model_list = [\n    (\"features_model8_2.pkl\", \"reg_model8_fold<NUMBER>_2.pkl\"),\n    (\"features_model8_3.pkl\", \"reg_model8_fold<NUMBER>_3.pkl\"),\n    (\"features_model8_4.pkl\", \"reg_model8_fold<NUMBER>_4.pkl\"),\n    (\"features_model8_5.pkl\", \"reg_model8_fold<NUMBER>_5.pkl\"),\n    (\"features_model8_6.pkl\", \"reg_model8_fold<NUMBER>_6.pkl\"),\n    (\"features_model8_7.pkl\", \"reg_model8_fold<NUMBER>_7.pkl\"),\n    (\"features_model8_8.pkl\", \"reg_model8_fold<NUMBER>_8.pkl\"),\n]\n\nlist_of_features = []\nfor feature_names_file, _ in model_list:\n    feature_names = pickle.load(open(f\"../input/adc24-checkpoints/{feature_names_file}\", 'rb'))\n    list_of_features.extend(feature_names)\n\nfeature_names = list(set(list_of_features))\n","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:13:30.467107Z","iopub.execute_input":"2024-10-18T07:13:30.46813Z","iopub.status.idle":"2024-10-18T07:13:30.478054Z","shell.execute_reply.started":"2024-10-18T07:13:30.468081Z","shell.execute_reply":"2024-10-18T07:13:30.477145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_ratio, partial_ratio, transit_limiting_timestamps = get_ratio(feature_names, pre_train)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:13:32.326497Z","iopub.execute_input":"2024-10-18T07:13:32.326893Z","iopub.status.idle":"2024-10-18T07:14:21.728077Z","shell.execute_reply.started":"2024-10-18T07:13:32.326855Z","shell.execute_reply":"2024-10-18T07:14:21.727171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = pd.DataFrame()\nif len(total_ratio) > 0:\n    features = pd.DataFrame(total_ratio, index=adc_info.index)\n\nif len(partial_ratio) > 0:\n    partial = pd.DataFrame(partial_ratio, index=adc_info.index)\n    if features.empty:\n        features = partial\n    else:\n        features = features.join(partial)\n\nfeatures.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:17:54.139697Z","iopub.execute_input":"2024-10-18T07:17:54.140118Z","iopub.status.idle":"2024-10-18T07:17:54.179646Z","shell.execute_reply.started":"2024-10-18T07:17:54.140066Z","shell.execute_reply":"2024-10-18T07:17:54.178444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.linear_model import LinearRegression\nimport pickle\n\naggregated_predictions = []\nfor feature_names_file, model_name in model_list:\n    feature_names = pickle.load(open(f\"../input/adc24-checkpoints/{feature_names_file}\", 'rb'))\n    Nfold = 5\n    predictions = np.zeros((features.shape[0], Nfold))\n    for fold in range(Nfold):\n        _name = model_name.replace(\"<NUMBER>\", str(fold)) \n        reg = pickle.load(open(f\"../input/adc24-checkpoints/{_name}\", 'rb'))\n        predictions[:, fold] = reg.predict(features[feature_names])\n    all_s = predictions.mean(axis=1)\n    aggregated_predictions.append(all_s)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:19:25.729708Z","iopub.execute_input":"2024-10-18T07:19:25.730333Z","iopub.status.idle":"2024-10-18T07:19:25.768383Z","shell.execute_reply.started":"2024-10-18T07:19:25.730291Z","shell.execute_reply":"2024-10-18T07:19:25.767675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_s = np.array(aggregated_predictions).mean(axis=0)\nall_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))","metadata":{"execution":{"iopub.status.busy":"2024-10-18T07:19:47.401148Z","iopub.execute_input":"2024-10-18T07:19:47.40211Z","iopub.status.idle":"2024-10-18T07:19:47.407531Z","shell.execute_reply.started":"2024-10-18T07:19:47.402056Z","shell.execute_reply":"2024-10-18T07:19:47.406451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create submission file","metadata":{}},{"cell_type":"code","source":"def postprocessing(pred_array, index, sigma_pred):\n    \"\"\"Create a submission dataframe from its components\n    \n    Parameters:\n    pred_array: ndarray of shape (n_samples, 283)\n    index: pandas.Index of length n_samples with name 'planet_id'\n    sigma_pred: float\n    \n    Return value:\n    df: DataFrame of shape (n_samples, 566) with planet_id as index\n    \"\"\"\n    wavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')\n    df = pd.concat(\n        [pd.DataFrame(pred_array.clip(0, None), index=index, columns=wavelengths.columns),\n         pd.DataFrame(sigma_pred, index=index, columns=[f\"sigma_{i}\" for i in range(1, 284)])],\n         axis=1\n    )\n    df.index.name = \"planet_id\"\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-10-17T12:51:56.232093Z","iopub.execute_input":"2024-10-17T12:51:56.232477Z","iopub.status.idle":"2024-10-17T12:51:56.23892Z","shell.execute_reply.started":"2024-10-17T12:51:56.232434Z","shell.execute_reply":"2024-10-17T12:51:56.238031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_submission_file(predictions_valid, predictions_std_valid, index, submission_file_index): \n    wavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')\n    submit = postprocessing(predictions_valid, index, predictions_std_valid)\n    display(submit.sample(min(3, submit.shape[0])))\n    submit.to_csv(f'submission_{submission_file_index}.csv')\n\n# if not DEBUG: \n#     create_submission_file(predictions_valid, predictions_std_valid, index)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T12:51:57.048832Z","iopub.execute_input":"2024-10-17T12:51:57.049567Z","iopub.status.idle":"2024-10-17T12:51:57.055111Z","shell.execute_reply.started":"2024-10-17T12:51:57.049527Z","shell.execute_reply":"2024-10-17T12:51:57.054029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sigma = np.ones(shape=(all_s.shape[0], all_s.shape[1]))\nsigma = sigma * 1.0e-4\nscale = 1.00000000000\ncreate_submission_file(all_s * scale, sigma, adc_info.index, 0)\nsubmit = pd.read_csv(f'submission_{0}.csv', index_col=0)\n\nif DEBUG:\n    test_sub_file_name = \"sub_file_emulation.csv\"\nelse:\n    test_sub_file_name = \"submission.csv\"\n# submit.to_csv(test_sub_file_name)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T12:51:57.475652Z","iopub.execute_input":"2024-10-17T12:51:57.475971Z","iopub.status.idle":"2024-10-17T12:51:57.57717Z","shell.execute_reply.started":"2024-10-17T12:51:57.475938Z","shell.execute_reply":"2024-10-17T12:51:57.576498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit = submit.join(adc_info[[\"star\"]])\n\nstar0 = submit[submit[\"star\"] == 0]\nstar1 = submit[submit[\"star\"] == 1]\nstar2 = submit[~submit[\"star\"].isin([0, 1])]\n\nfactor = {0: 1.25, 1: 0.7, 2: 1.25}\nnew_sub = []\nfor i, star in enumerate([star0, star1, star2]):\n    a = star[[f'wl_{c}' for c in range(1, 284)]] \n    b = star[[f'sigma_{c}' for c in range(1, 284)]] * factor[i]\n    new_sub.append(pd.concat([a, b], axis=1))\nnew_sub = pd.concat(new_sub)  \nnew_sub = new_sub.loc[adc_info.index]\nnew_sub.to_csv(test_sub_file_name)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T12:51:58.688135Z","iopub.execute_input":"2024-10-17T12:51:58.688536Z","iopub.status.idle":"2024-10-17T12:51:58.744346Z","shell.execute_reply.started":"2024-10-17T12:51:58.688496Z","shell.execute_reply":"2024-10-17T12:51:58.743608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    preds = pd.read_csv(\"sub_file_emulation.csv\", index_col=0)\n    targets = pd.read_csv(f'{path_folder}/train_labels.csv', index_col=0)\n    overall_score = np.mean((preds[targets.columns].values - targets.loc[preds.index].values)**2)**0.5 * 1E+6\n    print(f\"RMSE :  {overall_score:9.3f} ppm\")","metadata":{"execution":{"iopub.status.busy":"2024-10-17T12:51:59.512834Z","iopub.execute_input":"2024-10-17T12:51:59.513202Z","iopub.status.idle":"2024-10-17T12:51:59.604693Z","shell.execute_reply.started":"2024-10-17T12:51:59.513166Z","shell.execute_reply":"2024-10-17T12:51:59.603022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}