{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":191155259,"sourceType":"kernelVersion"},{"sourceId":196522559,"sourceType":"kernelVersion"}],"dockerImageVersionId":30761,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"LAUNCH_VARIANT,ENSEMBLE_SOLUTIONS = 'option 89',['SOLUTION_11','SOLUTION_9']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\nimport scipy.stats\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import r2_score, mean_squared_error\nimport itertools\nfrom scipy.optimize import minimize\nfrom functools import partial\nimport random, os\nfrom astropy.stats import sigma_clip","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/test_adc_info.csv',\n                           index_col='planet_id')\naxis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_linear_corr(linear_corr,clean_signal):\n    linear_corr = np.flip(linear_corr, axis=0)\n    for x, y in itertools.product(\n                range(clean_signal.shape[1]), range(clean_signal.shape[2])\n            ):\n        poli = np.poly1d(linear_corr[:, x, y])\n        clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n    return clean_signal\n\ndef clean_dark(signal, dark, dt):\n    dark = np.tile(dark, (signal.shape[0], 1, 1))\n    signal -= dark* dt[:, np.newaxis, np.newaxis]\n    return signal\n\ndef preproc(dataset, adc_info, sensor, binning = 15):\n    cut_inf, cut_sup = 39, 321\n    sensor_sizes_dict = {\"AIRS-CH0\":[[11250, 32, 356], [1, 32, cut_sup-cut_inf]], \"FGS1\":[[135000, 32, 32], [1, 32, 32]]}\n    binned_dict = {\"AIRS-CH0\":[11250 // binning // 2, 282], \"FGS1\":[135000 // binning // 2]}\n    linear_corr_dict = {\"AIRS-CH0\":(6, 32, 356), \"FGS1\":(6, 32, 32)}\n    planet_ids = adc_info.index\n    \n    feats = []\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        signal = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/{sensor}_signal.parquet').to_numpy()\n        dark_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dark.parquet', engine='pyarrow').to_numpy()\n        dead_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dead.parquet', engine='pyarrow').to_numpy()\n        flat_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/flat.parquet', engine='pyarrow').to_numpy()\n        linear_corr = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/linear_corr.parquet').values.astype(np.float64).reshape(linear_corr_dict[sensor])\n\n        signal = signal.reshape(sensor_sizes_dict[sensor][0]) \n        gain = adc_info[f'{sensor}_adc_gain'].values[i]\n        offset = adc_info[f'{sensor}_adc_offset'].values[i]\n        signal = signal / gain + offset\n        \n        hot = sigma_clip(\n            dark_frame, sigma=5, maxiters=5\n        ).mask\n        \n        if sensor != \"FGS1\":\n            signal = signal[:, :, cut_inf:cut_sup] #11250 * 32 * 282\n            #dt = axis_info['AIRS-CH0-integration_time'].dropna().values\n            dt = np.ones(len(signal))*0.1 \n            dt[1::2] += 4.5 #@bilzard idea\n            linear_corr = linear_corr[:, :, cut_inf:cut_sup]\n            dark_frame = dark_frame[:, cut_inf:cut_sup]\n            dead_frame = dead_frame[:, cut_inf:cut_sup]\n            flat_frame = flat_frame[:, cut_inf:cut_sup]\n            hot = hot[:, cut_inf:cut_sup]\n        else:\n            dt = np.ones(len(signal))*0.1\n            dt[1::2] += 0.1\n            \n        signal = signal.clip(0) #@graySnow idea\n        linear_corr_signal = apply_linear_corr(linear_corr, signal)\n        signal = clean_dark(linear_corr_signal, dark_frame, dt)\n        \n        flat = flat_frame.reshape(sensor_sizes_dict[sensor][1])\n        flat[dead_frame.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n        flat[hot.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n        signal = signal / flat\n        \n        if sensor == \"FGS1\":\n            signal = signal.reshape((sensor_sizes_dict[sensor][0][0], sensor_sizes_dict[sensor][0][1]*sensor_sizes_dict[sensor][0][2]))\n        \n        mean_signal = np.nanmean(signal, axis=1) # mean over the 32*32(FGS1) or 32(CH0) pixels\n        cds_signal = (mean_signal[1::2] - mean_signal[0::2])\n        \n        binned = np.zeros((binned_dict[sensor]))\n        for j in range(cds_signal.shape[0] // binning):\n            binned[j] = cds_signal[j*binning:j*binning+binning].mean(axis=0)\n                   \n        if sensor == \"FGS1\":\n            binned = binned.reshape((binned.shape[0],1))\n            \n        feats.append(binned)\n        \n    return np.stack(feats)\n    \npre_train = np.concatenate([preproc('test', test_adc_info, \"FGS1\", 30*12), preproc('test', test_adc_info, \"AIRS-CH0\", 30)], axis=2)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def phase_detector(signal):\n    phase1, phase2 = None, None\n    best_drop = 0\n    for i in range(50//2,150//2):        \n        t1 = signal[i:i+20//2].max() - signal[i:i+20//2].min()\n        if t1 > best_drop:\n            phase1 = i+(20+5)//2\n            best_drop = t1\n    \n    best_drop = 0\n    for i in range(200//2,250//2):\n        t1 = signal[i:i+20//2].max() - signal[i:i+20//2].min()\n        if t1 > best_drop:\n            phase2 = i-5//2\n            best_drop = t1\n    \n    return phase1, phase2\n\ndef try_s(signal, p1, p2, deg, s):\n    out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n    x, y = out, signal[out].tolist()\n    x = x + list(range(p1,p2))\n\n    y = y + (signal[p1:p2] * (1 + s[0])).tolist()\n    z = np.polyfit(x, y, deg)\n    p = np.poly1d(z)\n    q = np.abs(p(x) - y).mean()\n\n    if s < 1e-4:\n        return q + 1e3\n\n    return q\n    \ndef calibrate_signal(signal):\n    p1,p2 = phase_detector(signal)\n\n    best_deg, best_score = 1, 1e12\n    for deg in range(1, 6):\n        f = partial(try_s, signal, p1, p2, deg)\n        r = minimize(f, [0.001], method = 'Nelder-Mead')\n        s = r.x[0]\n\n        out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n        x, y = out, signal[out].tolist()\n        x = x + list(range(p1,p2))\n        y = y + (signal[p1:p2] * (1 + s)).tolist()\n    \n        z = np.polyfit(x, y, deg)\n        p = np.poly1d(z)\n        q = np.abs(p(x) - y).mean()\n        \n        if q < best_score:\n            best_score = q\n            best_deg = deg\n        \n        print(deg, q)\n            \n    z = np.polyfit(x, y, best_deg)\n    p = np.poly1d(z)\n\n    return s, x, y, p(x)\n\ndef calibrate_train(signal):\n    p1,p2 = phase_detector(signal)\n    \n    best_deg, best_score = 1, 1e12\n    for deg in range(1, 6):\n        f = partial(try_s, signal, p1, p2, deg)\n        r = minimize(f, [0.001], method = 'Nelder-Mead')\n        s = r.x[0]\n\n        out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n        x, y = out, signal[out].tolist()\n        x = x + list(range(p1,p2))\n        y = y + (signal[p1:p2] * (1 + s)).tolist()\n    \n        z = np.polyfit(x, y, deg)\n        p = np.poly1d(z)\n        q = np.abs(p(x) - y).mean()\n        \n        if q < best_score:\n            best_score = q\n            best_deg = deg\n            \n    z = np.polyfit(x, y, best_deg)\n    p = np.poly1d(z)\n    \n    return s, p(np.arange(signal.shape[0])), p1, p2\n\n\ntrain = pre_train.copy()\nall_s = []\nfor i in range(len(test_adc_info)):\n    signal = train[i,:,1:].mean(axis=1)\n    s, p, p1, p2 = calibrate_train(pre_train[i,:,1:].mean(axis=1))\n    all_s.append(s)\n        \n#copy answer 283 times because we predict mean value\ntrain_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))        \ntrain_sigma = np.ones_like(train_s) * 0.000176","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = 0\ns, x, y, y_new = calibrate_signal(pre_train[n,:,1:].mean(axis=1))\nplt.scatter(x,y)\nplt.scatter(x,y_new)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/sample_submission.csv')\n\npreds = train_s.clip(0)\nsigmas = train_sigma\nsubmission = pd.DataFrame(np.concatenate([preds,sigmas], axis=1), columns=ss.columns[1:])\nsubmission.index = test_adc_info.index\nsubmission.to_csv('submission_2.csv')","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_7' in ENSEMBLE_SOLUTIONS:\n    \n    import pandas as pd\n    import matplotlib.pyplot as plt\n    import numpy as np\n    import seaborn as sns\n    import scipy.stats\n    from tqdm import tqdm\n\n    from sklearn.model_selection import cross_val_predict\n    from sklearn.linear_model import Ridge\n    from sklearn.metrics import r2_score, mean_squared_error\n    import itertools\n    from scipy.optimize import minimize\n    from functools import partial\n    import random, os\n    from astropy.stats import sigma_clip","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_7' in ENSEMBLE_SOLUTIONS:\n    \n    test_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/test_adc_info.csv',\n                               index_col='planet_id')\n    axis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_7' in ENSEMBLE_SOLUTIONS:\n    \n    def apply_linear_corr(linear_corr,clean_signal):\n        linear_corr = np.flip(linear_corr, axis=0)\n        for x, y in itertools.product(\n                    range(clean_signal.shape[1]), range(clean_signal.shape[2])\n                ):\n            poli = np.poly1d(linear_corr[:, x, y])\n            clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n        return clean_signal\n\n    def clean_dark(signal, dark, dt):\n        dark = np.tile(dark, (signal.shape[0], 1, 1))\n        signal -= dark* dt[:, np.newaxis, np.newaxis]\n        return signal\n\n    def preproc(dataset, adc_info, sensor, binning = 15):\n        cut_inf, cut_sup = 39, 321\n        sensor_sizes_dict = {\"AIRS-CH0\":[[11250, 32, 356], [1, 32, cut_sup-cut_inf]], \"FGS1\":[[135000, 32, 32], [1, 32, 32]]}\n        binned_dict = {\"AIRS-CH0\":[11250 // binning // 2, 282], \"FGS1\":[135000 // binning // 2]}\n        linear_corr_dict = {\"AIRS-CH0\":(6, 32, 356), \"FGS1\":(6, 32, 32)}\n        planet_ids = adc_info.index\n\n        feats = []\n        for i, planet_id in tqdm(list(enumerate(planet_ids))):\n            signal = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/{sensor}_signal.parquet').to_numpy()\n            dark_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dark.parquet', engine='pyarrow').to_numpy()\n            dead_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dead.parquet', engine='pyarrow').to_numpy()\n            flat_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/flat.parquet', engine='pyarrow').to_numpy()\n            linear_corr = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/linear_corr.parquet').values.astype(np.float64).reshape(linear_corr_dict[sensor])\n\n            signal = signal.reshape(sensor_sizes_dict[sensor][0]) \n            gain = adc_info[f'{sensor}_adc_gain'].values[i]\n            offset = adc_info[f'{sensor}_adc_offset'].values[i]\n            signal = signal / gain + offset\n\n            hot = sigma_clip(\n                dark_frame, sigma=8, maxiters=5\n            ).mask\n\n            if sensor != \"FGS1\":\n                signal = signal[:, :, cut_inf:cut_sup] #11250 * 32 * 282\n                #dt = axis_info['AIRS-CH0-integration_time'].dropna().values\n                dt = np.ones(len(signal))*0.1 \n                dt[1::2] += 4.5 #@bilzard idea\n                linear_corr = linear_corr[:, :, cut_inf:cut_sup]\n                dark_frame = dark_frame[:, cut_inf:cut_sup]\n                dead_frame = dead_frame[:, cut_inf:cut_sup]\n                flat_frame = flat_frame[:, cut_inf:cut_sup]\n                hot = hot[:, cut_inf:cut_sup]\n            else:\n                dt = np.ones(len(signal))*0.1\n                dt[1::2] += 0.1\n\n            signal = signal.clip(0) #@graySnow idea\n            linear_corr_signal = apply_linear_corr(linear_corr, signal)\n            signal = clean_dark(linear_corr_signal, dark_frame, dt)\n\n            flat = flat_frame.reshape(sensor_sizes_dict[sensor][1])\n            flat[dead_frame.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n            flat[hot.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n            signal = signal / flat\n\n            if sensor == \"FGS1\":\n                signal = signal.reshape((sensor_sizes_dict[sensor][0][0], sensor_sizes_dict[sensor][0][1]*sensor_sizes_dict[sensor][0][2]))\n\n            mean_signal = np.nanmean(signal, axis=1) # mean over the 32*32(FGS1) or 32(CH0) pixels\n            cds_signal = (mean_signal[1::2] - mean_signal[0::2])\n\n            binned = np.zeros((binned_dict[sensor]))\n            for j in range(cds_signal.shape[0] // binning):\n                binned[j] = cds_signal[j*binning:j*binning+binning].mean(axis=0)\n\n            if sensor == \"FGS1\":\n                binned = binned.reshape((binned.shape[0],1))\n\n            feats.append(binned)\n\n        return np.stack(feats)\n\n    pre_train = np.concatenate([preproc('test', test_adc_info, \"FGS1\", 30*12), preproc('test', test_adc_info, \"AIRS-CH0\", 30)], axis=2)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_7' in ENSEMBLE_SOLUTIONS:\n    \n    def phase_detector(signal):\n        phase1, phase2 = None, None\n        best_drop = 0\n        for i in range(50//2,150//2):        \n            t1 = signal[i:i+20//2].max() - signal[i:i+20//2].min()\n            if t1 > best_drop:\n                phase1 = i+(20+5)//2\n                best_drop = t1\n\n        best_drop = 0\n        for i in range(200//2,250//2):\n            t1 = signal[i:i+20//2].max() - signal[i:i+20//2].min()\n            if t1 > best_drop:\n                phase2 = i-5//2\n                best_drop = t1\n\n        return phase1, phase2\n\n    def try_s(signal, p1, p2, deg, s):\n        out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n        x, y = out, signal[out].tolist()\n        x = x + list(range(p1,p2))\n\n        y = y + (signal[p1:p2] * (1 + s[0])).tolist()\n        z = np.polyfit(x, y, deg)\n        p = np.poly1d(z)\n        q = np.abs(p(x) - y).mean()\n\n        if s < 1e-4:\n            return q + 1e3\n\n        return q\n\n    def calibrate_signal(signal):\n        p1,p2 = phase_detector(signal)\n\n        best_deg, best_score = 1, 1e12\n        for deg in range(1, 6):\n            f = partial(try_s, signal, p1, p2, deg)\n            r = minimize(f, [0.001], method = 'Nelder-Mead')\n            s = r.x[0]\n\n            out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n            x, y = out, signal[out].tolist()\n            x = x + list(range(p1,p2))\n            y = y + (signal[p1:p2] * (1 + s)).tolist()\n\n            z = np.polyfit(x, y, deg)\n            p = np.poly1d(z)\n            q = np.abs(p(x) - y).mean()\n\n            if q < best_score:\n                best_score = q\n                best_deg = deg\n\n            print(deg, q)\n\n        z = np.polyfit(x, y, best_deg)\n        p = np.poly1d(z)\n\n        return s, x, y, p(x)\n\n    def calibrate_train(signal):\n        p1,p2 = phase_detector(signal)\n\n        best_deg, best_score = 1, 1e12\n        for deg in range(1, 6):\n            f = partial(try_s, signal, p1, p2, deg)\n            r = minimize(f, [0.0001], method = 'Nelder-Mead')\n            s = r.x[0]\n\n            out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n            x, y = out, signal[out].tolist()\n            x = x + list(range(p1,p2))\n            y = y + (signal[p1:p2] * (1 + s)).tolist()\n\n            z = np.polyfit(x, y, deg)\n            p = np.poly1d(z)\n            q = np.abs(p(x) - y).mean()\n\n            if q < best_score:\n                best_score = q\n                best_deg = deg\n\n        z = np.polyfit(x, y, best_deg)\n        p = np.poly1d(z)\n\n        return s, p(np.arange(signal.shape[0])), p1, p2\n\n\n    train = pre_train.copy()\n    all_s = []\n    for i in range(len(test_adc_info)):\n        signal = train[i,:,1:].mean(axis=1)\n        s, p, p1, p2 = calibrate_train(pre_train[i,:,1:].mean(axis=1))\n        all_s.append(s)\n\n    #copy answer 283 times because we predict mean value\n    train_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))        \n    train_sigma = np.ones_like(train_s) * 0.000176","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_7' in ENSEMBLE_SOLUTIONS:\n    \n    n = 0\n    s, x, y, y_new = calibrate_signal(pre_train[n,:,1:].mean(axis=1))\n    plt.scatter(x,y)\n    plt.scatter(x,y_new)","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_7' in ENSEMBLE_SOLUTIONS:    \n    \n    ss = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/sample_submission.csv')\n\n    preds = train_s.clip(0)\n    sigmas = train_sigma\n    submission = pd.DataFrame(np.concatenate([preds,sigmas], axis=1), columns=ss.columns[1:])\n    submission.index = test_adc_info.index\n    submission.to_csv('submission_7.csv')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_9' in ENSEMBLE_SOLUTIONS:\n    \n    import pandas as pd\n    import matplotlib.pyplot as plt\n    import numpy as np\n    from tqdm import tqdm\n    import joblib\n\n    from sklearn.linear_model import Ridge\n    from sklearn.metrics import r2_score, mean_squared_error\n    import itertools\n\n    from scipy.optimize import minimize\n    from scipy import optimize\n\n    from astropy.stats import sigma_clip","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_9' in ENSEMBLE_SOLUTIONS:\n    \n    dataset = 'test'\n    adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/'+f'{dataset}_adc_info.csv',index_col='planet_id')\n    axis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_9' in ENSEMBLE_SOLUTIONS:\n    \n    def apply_linear_corr(linear_corr,clean_signal):\n        linear_corr = np.flip(linear_corr, axis=0)\n        for x, y in itertools.product(\n                    range(clean_signal.shape[1]), range(clean_signal.shape[2])\n                ):\n            poli = np.poly1d(linear_corr[:, x, y])\n            clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n        return clean_signal\n\n    def clean_dark(signal, dark, dt):\n        dark = np.tile(dark, (signal.shape[0], 1, 1))\n        signal -= dark* dt[:, np.newaxis, np.newaxis]\n        return signal\n\n    def preproc(dataset, adc_info, sensor, binning = 15):\n        cut_inf, cut_sup = 39, 321\n        sensor_sizes_dict = {\"AIRS-CH0\":[[11250, 32, 356], [1, 32, cut_sup-cut_inf]], \"FGS1\":[[135000, 32, 32], [1, 32, 32]]}\n        binned_dict = {\"AIRS-CH0\":[11250 // binning // 2, 282], \"FGS1\":[135000 // binning // 2]}\n        linear_corr_dict = {\"AIRS-CH0\":(6, 32, 356), \"FGS1\":(6, 32, 32)}\n        planet_ids = adc_info.index\n\n        feats = []\n        for i, planet_id in tqdm(list(enumerate(planet_ids))):\n            signal = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/{sensor}_signal.parquet').to_numpy()\n            dark_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dark.parquet', engine='pyarrow').to_numpy()\n            dead_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dead.parquet', engine='pyarrow').to_numpy()\n            flat_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/flat.parquet', engine='pyarrow').to_numpy()\n            linear_corr = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/linear_corr.parquet').values.astype(np.float64).reshape(linear_corr_dict[sensor])\n\n            signal = signal.reshape(sensor_sizes_dict[sensor][0]) \n            gain = adc_info[f'{sensor}_adc_gain'].values[i]\n            offset = adc_info[f'{sensor}_adc_offset'].values[i]\n            signal = signal / gain + offset\n\n            hot = sigma_clip(\n                dark_frame, sigma=5, maxiters=5\n            ).mask\n\n            if sensor != \"FGS1\":\n                signal = signal[:, :, cut_inf:cut_sup] \n                dt = np.ones(len(signal))*0.1 \n                dt[1::2] += 4.5 #@bilzard idea\n                linear_corr = linear_corr[:, :, cut_inf:cut_sup]\n                dark_frame = dark_frame[:, cut_inf:cut_sup]\n                dead_frame = dead_frame[:, cut_inf:cut_sup]\n                flat_frame = flat_frame[:, cut_inf:cut_sup]\n                hot = hot[:, cut_inf:cut_sup]\n            else:\n                dt = np.ones(len(signal))*0.1\n                dt[1::2] += 0.1\n\n            signal = signal.clip(0) #@graySnow idea\n            linear_corr_signal = apply_linear_corr(linear_corr, signal)\n            signal = clean_dark(linear_corr_signal, dark_frame, dt)\n\n            flat = flat_frame.reshape(sensor_sizes_dict[sensor][1])\n            flat[dead_frame.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n            flat[hot.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n            signal = signal / flat\n\n\n            if sensor == \"FGS1\":\n                signal = signal[:,10:22,10:22] # **** updates ****\n                signal = signal.reshape(sensor_sizes_dict[sensor][0][0],144) # # **** updates ****\n\n            if sensor != \"FGS1\":\n                signal = signal[:,10:22,:] # **** updates ****\n\n            mean_signal = np.nanmean(signal, axis=1) \n            cds_signal = (mean_signal[1::2] - mean_signal[0::2])\n\n            binned = np.zeros((binned_dict[sensor]))\n            for j in range(cds_signal.shape[0] // binning):\n                binned[j] = cds_signal[j*binning:j*binning+binning].mean(axis=0) \n\n            if sensor == \"FGS1\":\n                binned = binned.reshape((binned.shape[0],1))\n\n            feats.append(binned)\n\n        return np.stack(feats)\n\n    pre_train = np.concatenate([preproc(f'{dataset}', adc_info, \"FGS1\", 30*12), preproc(f'{dataset}', adc_info, \"AIRS-CH0\", 30)], axis=2)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_9' in ENSEMBLE_SOLUTIONS:\n    \n    def phase_detector(signal):\n\n        MIN = np.argmin(signal[30:140])+30\n        signal1 = signal[:MIN ]\n        signal2 = signal[MIN :]\n\n        first_derivative1 = np.gradient(signal1)\n        first_derivative1 /= first_derivative1.max()\n        first_derivative2 = np.gradient(signal2)\n        first_derivative2 /= first_derivative2.max()\n\n        phase1 = np.argmin(first_derivative1)\n        phase2 = np.argmax(first_derivative2) + MIN\n\n        return phase1, phase2\n\n    def objective(s):\n\n        best_q = 1e10\n        for i in range(4) :\n            delta = 2\n            x = list(range(signal.shape[0]-delta*4))\n            y = signal[:p1-delta].tolist() + (signal[p1+delta:p2 - delta] * (1 + s)).tolist() + signal[p2+delta:].tolist()\n\n            z = np.polyfit(x, y, deg=i)\n            p = np.poly1d(z)\n            q = np.abs(p(x) - y).mean()\n\n        if q < best_q :\n            best_q = q\n\n        return q\n\n\n    all_s = []\n    for i in tqdm(range(len(adc_info))):\n\n        signal = pre_train[i,:,1:].mean(axis=1)\n        p1,p2 = phase_detector(signal)\n\n        r = minimize(\n                    objective,\n                    [0.0001],\n                    method= 'Nelder-Mead'\n                      )\n        s = r.x[0]\n        all_s.append(s)\n\n    all_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))        ","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_9' in ENSEMBLE_SOLUTIONS:\n    \n    ss = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/sample_submission.csv')\n\n    __k = 0.000140985 # 0.0001422\n\n    sigma = np.ones_like(all_s) * __k\n    pred = all_s.clip(0) \n    submission = pd.DataFrame(np.concatenate([pred,sigma], axis=1), columns=ss.columns[1:])\n    submission.index = adc_info.index\n    submission.to_csv('submission_9.csv')\n    submission","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_10' in ENSEMBLE_SOLUTIONS:\n\n    import pandas as pd\n    import matplotlib.pyplot as plt\n    import numpy as np\n    from tqdm import tqdm\n    import joblib\n    from sklearn.linear_model import Ridge\n    from sklearn.metrics import r2_score, mean_squared_error\n    import itertools\n    from scipy.optimize import minimize\n    from scipy import optimize\n    from astropy.stats import sigma_clip\n\n    dataset = 'test'\n    adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/'+f'{dataset}_adc_info.csv',index_col='planet_id')\n    axis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_10' in ENSEMBLE_SOLUTIONS:\n    \n    def apply_linear_corr(linear_corr,clean_signal):\n        linear_corr = np.flip(linear_corr, axis=0)\n        for x, y in itertools.product(\n                    range(clean_signal.shape[1]), range(clean_signal.shape[2])\n                ):\n            poli = np.poly1d(linear_corr[:, x, y])\n            clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n        return clean_signal\n\n    def clean_dark(signal, dark, dt):\n        dark = np.tile(dark, (signal.shape[0], 1, 1))\n        signal -= dark* dt[:, np.newaxis, np.newaxis]\n        return signal\n\n    def preproc(dataset, adc_info, sensor, binning = 15):\n        cut_inf, cut_sup = 39, 321\n        sensor_sizes_dict = {\"AIRS-CH0\":[[11250, 32, 356], [1, 32, cut_sup-cut_inf]], \"FGS1\":[[135000, 32, 32], [1, 32, 32]]}\n        binned_dict = {\"AIRS-CH0\":[11250 // binning // 2, 282], \"FGS1\":[135000 // binning // 2]}\n        linear_corr_dict = {\"AIRS-CH0\":(6, 32, 356), \"FGS1\":(6, 32, 32)}\n        planet_ids = adc_info.index\n\n        feats = []\n        for i, planet_id in tqdm(list(enumerate(planet_ids))):\n            signal = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/{sensor}_signal.parquet').to_numpy()\n            dark_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dark.parquet', engine='pyarrow').to_numpy()\n            dead_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dead.parquet', engine='pyarrow').to_numpy()\n            flat_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/flat.parquet', engine='pyarrow').to_numpy()\n            linear_corr = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/linear_corr.parquet').values.astype(np.float64).reshape(linear_corr_dict[sensor])\n\n            signal = signal.reshape(sensor_sizes_dict[sensor][0]) \n            gain = adc_info[f'{sensor}_adc_gain'].values[i]\n            offset = adc_info[f'{sensor}_adc_offset'].values[i]\n            signal = signal / gain + offset\n\n            hot = sigma_clip(\n                dark_frame, sigma=5, maxiters=5\n            ).mask\n            \n            bilzard_idea = 4.5\n            \n            if LAUNCH_VARIANT == 'option 70': bilzard_idea = 4.15\n            if LAUNCH_VARIANT == 'option 71': bilzard_idea = 4.85\n            if LAUNCH_VARIANT == 'option 72': bilzard_idea = 5.15\n\n            if sensor != \"FGS1\":\n                signal = signal[:, :, cut_inf:cut_sup] \n                dt = np.ones(len(signal))*0.1 \n                dt[1::2] += bilzard_idea\n                linear_corr = linear_corr[:, :, cut_inf:cut_sup]\n                dark_frame = dark_frame[:, cut_inf:cut_sup]\n                dead_frame = dead_frame[:, cut_inf:cut_sup]\n                flat_frame = flat_frame[:, cut_inf:cut_sup]\n                hot = hot[:, cut_inf:cut_sup]\n            else:\n                dt = np.ones(len(signal))*0.1\n                dt[1::2] += 0.1\n\n            signal = signal.clip(0) #@graySnow idea\n            linear_corr_signal = apply_linear_corr(linear_corr, signal)\n            signal = clean_dark(linear_corr_signal, dark_frame, dt)\n\n            flat = flat_frame.reshape(sensor_sizes_dict[sensor][1])\n            flat[dead_frame.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n            flat[hot.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n            signal = signal / flat\n\n\n            if sensor == \"FGS1\":\n                signal = signal[:,10:22,10:22] # **** updates ****\n                signal = signal.reshape(sensor_sizes_dict[sensor][0][0],144) # # **** updates ****\n\n            if sensor != \"FGS1\":\n                signal = signal[:,10:22,:] # **** updates ****\n\n            mean_signal = np.nanmean(signal, axis=1) \n            cds_signal = (mean_signal[1::2] - mean_signal[0::2])\n\n            binned = np.zeros((binned_dict[sensor]))\n            for j in range(cds_signal.shape[0] // binning):\n                binned[j] = cds_signal[j*binning:j*binning+binning].mean(axis=0) \n\n            if sensor == \"FGS1\":\n                binned = binned.reshape((binned.shape[0],1))\n\n            feats.append(binned)\n\n        return np.stack(feats)\n\n    pre_train = np.concatenate([preproc(f'{dataset}', adc_info, \"FGS1\", 30*12), preproc(f'{dataset}', adc_info, \"AIRS-CH0\", 30)], axis=2)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_10' in ENSEMBLE_SOLUTIONS:\n    \n    def phase_detector(signal):\n\n        MIN = np.argmin(signal[30:140])+30\n        signal1 = signal[:MIN ]\n        signal2 = signal[MIN :]\n\n        first_derivative1 = np.gradient(signal1)\n        first_derivative1 /= first_derivative1.max()\n        first_derivative2 = np.gradient(signal2)\n        first_derivative2 /= first_derivative2.max()\n\n        phase1 = np.argmin(first_derivative1)\n        phase2 = np.argmax(first_derivative2) + MIN\n\n        return phase1, phase2\n\n    def objective(s):\n\n        best_q = 1e10\n        for i in range(4) :\n            delta = 2\n            x = list(range(signal.shape[0]-delta*4))\n            y = signal[:p1-delta].tolist() + (signal[p1+delta:p2 - delta] * (1 + s)).tolist() + signal[p2+delta:].tolist()\n\n            z = np.polyfit(x, y, deg=i)\n            p = np.poly1d(z)\n            q = np.abs(p(x) - y).mean()\n\n        if q < best_q :\n            best_q = q\n\n        return q\n    \n    \n    _k_min = 0.0001\n    \n    if LAUNCH_VARIANT == 'option 70': _k_min = 0.00015\n    if LAUNCH_VARIANT == 'option 71': _k_min = 0.000085\n    if LAUNCH_VARIANT == 'option 72': _k_min = 0.00021\n\n\n    all_s = []\n    for i in tqdm(range(len(adc_info))):\n\n        signal = pre_train[i,:,1:].mean(axis=1)\n        p1,p2 = phase_detector(signal)\n\n        r = minimize(\n                    objective,\n                    [0.00015],\n                    method= 'Nelder-Mead'\n                      )\n        s = r.x[0]\n        all_s.append(s)\n\n    all_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))        ","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_10' in ENSEMBLE_SOLUTIONS:\n    \n    ss = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/sample_submission.csv')\n\n    __k = 0.0001422\n\n    sigma = np.ones_like(all_s) * __k\n    pred = all_s.clip(0) \n    submission = pd.DataFrame(np.concatenate([pred,sigma], axis=1), columns=ss.columns[1:])\n    submission.index = adc_info.index\n    submission.to_csv('submission_10.csv')\n    submission","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_11' in ENSEMBLE_SOLUTIONS:\n    \n    import pandas as pd\n    import matplotlib.pyplot as plt\n    import numpy as np\n    import seaborn as sns\n    import scipy.stats\n    from tqdm import tqdm\n\n    from sklearn.model_selection import cross_val_predict\n    from sklearn.linear_model import Ridge\n    from sklearn.metrics import r2_score, mean_squared_error\n    import itertools\n    from scipy.optimize import minimize\n    from functools import partial\n    import random, os\n    from astropy.stats import sigma_clip","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_11' in ENSEMBLE_SOLUTIONS:\n    \n    test_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/test_adc_info.csv',\n                               index_col='planet_id')\n    axis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_11' in ENSEMBLE_SOLUTIONS:\n    \n    def apply_linear_corr(linear_corr,clean_signal):\n        linear_corr = np.flip(linear_corr, axis=0)\n        for x, y in itertools.product(\n                    range(clean_signal.shape[1]), range(clean_signal.shape[2])\n                ):\n            poli = np.poly1d(linear_corr[:, x, y])\n            clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n        return clean_signal\n\n    def clean_dark(signal, dark, dt):\n        dark = np.tile(dark, (signal.shape[0], 1, 1))\n        signal -= dark* dt[:, np.newaxis, np.newaxis]\n        return signal\n\n    def preproc(dataset, adc_info, sensor, binning = 15):\n        cut_inf, cut_sup = 39, 321\n        sensor_sizes_dict = {\"AIRS-CH0\":[[11250, 32, 356], [1, 32, cut_sup-cut_inf]], \"FGS1\":[[135000, 32, 32], [1, 32, 32]]}\n        binned_dict = {\"AIRS-CH0\":[11250 // binning // 2, 282], \"FGS1\":[135000 // binning // 2]}\n        linear_corr_dict = {\"AIRS-CH0\":(6, 32, 356), \"FGS1\":(6, 32, 32)}\n        planet_ids = adc_info.index\n\n        feats = []\n        for i, planet_id in tqdm(list(enumerate(planet_ids))):\n            signal = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/{sensor}_signal.parquet').to_numpy()\n            dark_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dark.parquet', engine='pyarrow').to_numpy()\n            dead_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dead.parquet', engine='pyarrow').to_numpy()\n            flat_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/flat.parquet', engine='pyarrow').to_numpy()\n            linear_corr = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/linear_corr.parquet').values.astype(np.float64).reshape(linear_corr_dict[sensor])\n\n            signal = signal.reshape(sensor_sizes_dict[sensor][0]) \n            gain = adc_info[f'{sensor}_adc_gain'].values[i]\n            offset = adc_info[f'{sensor}_adc_offset'].values[i]\n            signal = signal / gain + offset\n\n            hot = sigma_clip(\n                dark_frame, sigma=8, maxiters=5\n            ).mask\n\n            if sensor != \"FGS1\":\n                signal = signal[:, :, cut_inf:cut_sup] #11250 * 32 * 282\n                #dt = axis_info['AIRS-CH0-integration_time'].dropna().values\n                dt = np.ones(len(signal))*0.1 \n                dt[1::2] += 4.5 #@bilzard idea\n                linear_corr = linear_corr[:, :, cut_inf:cut_sup]\n                dark_frame = dark_frame[:, cut_inf:cut_sup]\n                dead_frame = dead_frame[:, cut_inf:cut_sup]\n                flat_frame = flat_frame[:, cut_inf:cut_sup]\n                hot = hot[:, cut_inf:cut_sup]\n            else:\n                dt = np.ones(len(signal))*0.1\n                dt[1::2] += 0.1\n\n            signal = signal.clip(0) #@graySnow idea\n            linear_corr_signal = apply_linear_corr(linear_corr, signal)\n            signal = clean_dark(linear_corr_signal, dark_frame, dt)\n\n            flat = flat_frame.reshape(sensor_sizes_dict[sensor][1])\n            flat[dead_frame.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n            flat[hot.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n            signal = signal / flat\n\n            if sensor == \"FGS1\":\n                signal = signal.reshape((sensor_sizes_dict[sensor][0][0], sensor_sizes_dict[sensor][0][1]*sensor_sizes_dict[sensor][0][2]))\n\n            mean_signal = np.nanmean(signal, axis=1) # mean over the 32*32(FGS1) or 32(CH0) pixels\n            cds_signal = (mean_signal[1::2] - mean_signal[0::2])\n\n            binned = np.zeros((binned_dict[sensor]))\n            for j in range(cds_signal.shape[0] // binning):\n                binned[j] = cds_signal[j*binning:j*binning+binning].mean(axis=0)\n\n            if sensor == \"FGS1\":\n                binned = binned.reshape((binned.shape[0],1))\n\n            feats.append(binned)\n\n        return np.stack(feats)\n\n    pre_train = np.concatenate([preproc('test', test_adc_info, \"FGS1\", 30*12), preproc('test', test_adc_info, \"AIRS-CH0\", 30)], axis=2)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_11' in ENSEMBLE_SOLUTIONS:\n    \n    def phase_detector(signal):\n        phase1, phase2 = None, None\n        best_drop = 0\n        for i in range(50//2,150//2):        \n            t1 = signal[i:i+20//2].max() - signal[i:i+20//2].min()\n            if t1 > best_drop:\n                phase1 = i+(20+5)//2\n                best_drop = t1\n\n        best_drop = 0\n        for i in range(200//2,250//2):\n            t1 = signal[i:i+20//2].max() - signal[i:i+20//2].min()\n            if t1 > best_drop:\n                phase2 = i-5//2\n                best_drop = t1\n\n        return phase1, phase2\n\n    def try_s(signal, p1, p2, deg, s):\n        out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n        x, y = out, signal[out].tolist()\n        x = x + list(range(p1,p2))\n\n        y = y + (signal[p1:p2] * (1 + s[0])).tolist()\n        z = np.polyfit(x, y, deg)\n        p = np.poly1d(z)\n        q = np.abs(p(x) - y).mean()\n\n        if s < 1e-4:\n            return q + 1e3\n\n        return q\n\n    def calibrate_signal(signal):\n        p1,p2 = phase_detector(signal)\n\n        best_deg, best_score = 1, 1e12\n        for deg in range(1, 4):\n            f = partial(try_s, signal, p1, p2, deg)\n            r = minimize(f, [0.001], method = 'Nelder-Mead')\n            s = r.x[0]\n\n            out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n            x, y = out, signal[out].tolist()\n            x = x + list(range(p1,p2))\n            y = y + (signal[p1:p2] * (1 + s)).tolist()\n\n            z = np.polyfit(x, y, deg)\n            p = np.poly1d(z)\n            q = np.abs(p(x) - y).mean()\n\n            if q < best_score:\n                best_score = q\n                best_deg = deg\n\n            print(deg, q)\n\n        z = np.polyfit(x, y, best_deg)\n        p = np.poly1d(z)\n\n        return s, x, y, p(x)\n\n    def calibrate_train(signal):\n        p1,p2 = phase_detector(signal)\n\n        best_deg, best_score = 1, 1e12\n        for deg in range(1, 4):\n            f = partial(try_s, signal, p1, p2, deg)\n            r = minimize(f, [0.0001], method = 'Nelder-Mead')\n            s = r.x[0]\n\n            out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n            x, y = out, signal[out].tolist()\n            x = x + list(range(p1,p2))\n            y = y + (signal[p1:p2] * (1 + s)).tolist()\n\n            z = np.polyfit(x, y, deg)\n            p = np.poly1d(z)\n            q = np.abs(p(x) - y).mean()\n\n            if q < best_score:\n                best_score = q\n                best_deg = deg\n\n        z = np.polyfit(x, y, best_deg)\n        p = np.poly1d(z)\n\n        return s, p(np.arange(signal.shape[0])), p1, p2\n\n\n    train = pre_train.copy()\n    all_s = []\n    for i in range(len(test_adc_info)):\n        signal = train[i,:,1:].mean(axis=1)\n        s, p, p1, p2 = calibrate_train(pre_train[i,:,1:].mean(axis=1))\n        all_s.append(s)\n\n    #copy answer 283 times because we predict mean value\n    train_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283)) \n    \n    _k_sigma = 0.000176\n    \n    if LAUNCH_VARIANT == 'option 87': _k_sigma = 0.000177\n    if LAUNCH_VARIANT == 'option 88': _k_sigma = 0.000178\n    if LAUNCH_VARIANT == 'option 89': _k_sigma = 0.000179\n    \n    train_sigma = np.ones_like(train_s) * _k_sigma","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_11' in ENSEMBLE_SOLUTIONS:\n    \n    n = 0\n    s, x, y, y_new = calibrate_signal(pre_train[n,:,1:].mean(axis=1))\n    plt.scatter(x,y)\n    plt.scatter(x,y_new)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'SOLUTION_11' in ENSEMBLE_SOLUTIONS:\n    \n    ss = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/sample_submission.csv')\n\n    preds = train_s.clip(0)\n    sigmas = train_sigma\n    submission = pd.DataFrame(np.concatenate([preds,sigmas], axis=1), columns=ss.columns[1:])\n    submission.index = test_adc_info.index\n    submission.to_csv('submission_11.csv')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble of solutions","metadata":{}},{"cell_type":"code","source":"if LAUNCH_VARIANT == 'option 87' or LAUNCH_VARIANT == 'option 88' or LAUNCH_VARIANT == 'option 89':\n    df_solution_x  = pd.read_csv('submission_11.csv')\n    df_solution_11 = pd.read_csv('submission_11.csv')\n    df_solution_9  = pd.read_csv('submission_9.csv')\n    df_solution_11 = df_solution_11.map(lambda x:x*0.0303215)\n    df_solution_9  = df_solution_9 .map(lambda x:x*0.9696785)\n    df_temp = df_solution_11.add(df_solution_9)\n    display(df_temp)\n    df_submission = df_temp.map(lambda x:x)\n    display(df_submission)\n    df_submission['planet_id'] = df_solution_x['planet_id']\n\n\nif LAUNCH_VARIANT == 'option 63':\n    df_solution_x = pd.read_csv('submission_7.csv')\n    df_solution_7 = pd.read_csv('submission_7.csv')\n    df_solution_9 = pd.read_csv('submission_9.csv')\n    df_solution_7 = df_solution_7.map(lambda x:x*0.0303215)\n    df_solution_9 = df_solution_9.map(lambda x:x*0.9696785)\n    df_temp = df_solution_7.add(df_solution_9)\n    display(df_temp)\n    df_submission = df_temp.map(lambda x:x)\n    display(df_submission)\n    df_submission['planet_id'] = df_solution_x['planet_id']","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv('submission.csv', index=False, float_format='%.8f')\ndf_submission","metadata":{},"execution_count":null,"outputs":[]}]}