{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":9417733,"sourceType":"datasetVersion","datasetId":5562609},{"sourceId":9469851,"sourceType":"datasetVersion","datasetId":5758676},{"sourceId":9498275,"sourceType":"datasetVersion","datasetId":5780132},{"sourceId":9498432,"sourceType":"datasetVersion","datasetId":5780234},{"sourceId":9618839,"sourceType":"datasetVersion","datasetId":5870398},{"sourceId":9666298,"sourceType":"datasetVersion","datasetId":5906387},{"sourceId":9670961,"sourceType":"datasetVersion","datasetId":5909830},{"sourceId":9672854,"sourceType":"datasetVersion","datasetId":5911260}],"dockerImageVersionId":30746,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\nimport scipy.stats\nfrom tqdm import tqdm\nimport pickle\n\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.linear_model import Ridge\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.metrics import r2_score, mean_squared_error\n\nfrom functools import partial\nfrom scipy.optimize import minimize\n\nfrom scipy.signal import argrelextrema\n\nfrom astropy.stats import sigma_clip\nimport itertools","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_adc_info.csv',\n                           index_col='planet_id')\ntest_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/test_adc_info.csv',\n                            index_col='planet_id')\ntrain_labels = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_labels.csv',\n                           index_col='planet_id')\nwavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')\naxis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"star0_indices = [train_adc_info.index.get_loc(i) for i in train_adc_info.query('star==0').index]\nstar1_indices = [train_adc_info.index.get_loc(i) for i in train_adc_info.query('star==1').index]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_linear_corr(linear_corr,clean_signal):\n    linear_corr = np.flip(linear_corr, axis=0)\n    for x, y in itertools.product(\n                range(clean_signal.shape[1]), range(clean_signal.shape[2])\n            ):\n        poli = np.poly1d(linear_corr[:, x, y])\n        clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n    return clean_signal\n\ndef clean_dark(signal, dark, dt):\n    dark = np.tile(dark, (signal.shape[0], 1, 1))\n    signal -= dark* dt[:, np.newaxis, np.newaxis]\n    return signal\n\ndef preproc(dataset, adc_info, sensor, binning = 15):\n    cut_inf, cut_sup = 39, 321\n    sensor_sizes_dict = {\"AIRS-CH0\":[[11250, 32, 356], [1, 32, cut_sup-cut_inf]], \"FGS1\":[[135000, 32, 32], [1, 32, 32]]}\n    binned_dict = {\"AIRS-CH0\":[11250 // binning // 2, 282], \"FGS1\":[135000 // binning // 2]}\n    linear_corr_dict = {\"AIRS-CH0\":(6, 32, 356), \"FGS1\":(6, 32, 32)}\n    planet_ids = adc_info.index\n    \n    feats = []\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        signal = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/{sensor}_signal.parquet').to_numpy()\n        dark_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dark.parquet', engine='pyarrow').to_numpy()\n        dead_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dead.parquet', engine='pyarrow').to_numpy()\n        flat_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/flat.parquet', engine='pyarrow').to_numpy()\n        linear_corr = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/linear_corr.parquet').values.astype(np.float64).reshape(linear_corr_dict[sensor])\n\n        signal = signal.reshape(sensor_sizes_dict[sensor][0]) \n        gain = adc_info[f'{sensor}_adc_gain'].values[i]\n        offset = adc_info[f'{sensor}_adc_offset'].values[i]\n        signal = signal / gain + offset\n        \n        hot = sigma_clip(\n            dark_frame, sigma=5, maxiters=5\n        ).mask\n        \n        if sensor != \"FGS1\":\n            signal = signal[:, :, cut_inf:cut_sup] \n            dt = np.ones(len(signal))*0.1 \n            dt[1::2] += 4.5 #@bilzard idea\n            linear_corr = linear_corr[:, :, cut_inf:cut_sup]\n            dark_frame = dark_frame[:, cut_inf:cut_sup]\n            dead_frame = dead_frame[:, cut_inf:cut_sup]\n            flat_frame = flat_frame[:, cut_inf:cut_sup]\n            hot = hot[:, cut_inf:cut_sup]\n        else:\n            dt = np.ones(len(signal))*0.1\n            dt[1::2] += 0.1\n            \n        signal = signal.clip(0) #@graySnow idea\n        linear_corr_signal = apply_linear_corr(linear_corr, signal)\n        signal = clean_dark(linear_corr_signal, dark_frame, dt)\n        \n        flat = flat_frame.reshape(sensor_sizes_dict[sensor][1])\n        flat[dead_frame.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n        flat[hot.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n        signal = signal / flat\n        \n        \n        if sensor == \"FGS1\":\n            signal = signal[:,10:22,10:22] # **** updates ****\n            signal = signal.reshape(sensor_sizes_dict[sensor][0][0],144) # # **** updates ****\n\n        if sensor != \"FGS1\":\n            signal = signal[:,10:22,:] # **** updates ****\n\n        mean_signal = np.nanmean(signal, axis=1) \n        cds_signal = (mean_signal[1::2] - mean_signal[0::2])\n        \n        binned = np.zeros((binned_dict[sensor]))\n        for j in range(cds_signal.shape[0] // binning):\n            binned[j] = cds_signal[j*binning:j*binning+binning].mean(axis=0)\n                    \n        if sensor == \"FGS1\":\n            binned = binned.reshape((binned.shape[0],1))\n        \n        feats.append(binned)\n        \n    return np.stack(feats)\n    \n#pre_train = np.concatenate([preproc('train', train_adc_info, \"FGS1\", 30*12), preproc('train', train_adc_info, \"AIRS-CH0\", 30)], axis=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preproc2(dataset, adc_info, sensor, binning = 15):\n    cut_inf, cut_sup = 39, 321\n    sensor_sizes_dict = {\"AIRS-CH0\":[[11250, 32, 356], [1, 32, cut_sup-cut_inf]], \"FGS1\":[[135000, 32, 32], [1, 32, 32]]}\n    binned_dict = {\"AIRS-CH0\":[11250 // binning // 2, 282], \"FGS1\":[135000 // binning // 2]}\n    linear_corr_dict = {\"AIRS-CH0\":(6, 32, 356), \"FGS1\":(6, 32, 32)}\n    planet_ids = adc_info.index\n    \n    feats = []\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        signal = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/{sensor}_signal.parquet').to_numpy()\n        dark_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dark.parquet', engine='pyarrow').to_numpy()\n        dead_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dead.parquet', engine='pyarrow').to_numpy()\n        flat_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/flat.parquet', engine='pyarrow').to_numpy()\n        linear_corr = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/linear_corr.parquet').values.astype(np.float64).reshape(linear_corr_dict[sensor])\n\n        signal = signal.reshape(sensor_sizes_dict[sensor][0]) \n        gain = adc_info[f'{sensor}_adc_gain'].values[i]\n        offset = adc_info[f'{sensor}_adc_offset'].values[i]\n        signal = signal / gain + offset\n        \n        hot = sigma_clip(\n            dark_frame, sigma=5, maxiters=5\n        ).mask\n        \n        if sensor != \"FGS1\":\n            signal = signal[:, :, cut_inf:cut_sup] #11250 * 32 * 282\n            #dt = axis_info['AIRS-CH0-integration_time'].dropna().values\n            dt = np.ones(len(signal))*0.1 \n            dt[1::2] += 4.5 #@bilzard idea\n            linear_corr = linear_corr[:, :, cut_inf:cut_sup]\n            dark_frame = dark_frame[:, cut_inf:cut_sup]\n            dead_frame = dead_frame[:, cut_inf:cut_sup]\n            flat_frame = flat_frame[:, cut_inf:cut_sup]\n            hot = hot[:, cut_inf:cut_sup]\n        else:\n            dt = np.ones(len(signal))*0.1\n            dt[1::2] += 0.1\n            \n        signal = signal.clip(0) #@graySnow idea\n        linear_corr_signal = apply_linear_corr(linear_corr, signal)\n        signal = clean_dark(linear_corr_signal, dark_frame, dt)\n        \n        flat = flat_frame.reshape(sensor_sizes_dict[sensor][1])\n        flat[dead_frame.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n        flat[hot.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n        signal = signal / flat\n        \n        if sensor == \"FGS1\":\n            signal = signal.reshape((sensor_sizes_dict[sensor][0][0], sensor_sizes_dict[sensor][0][1]*sensor_sizes_dict[sensor][0][2]))\n        #print(signal)\n        mean_signal = np.nanmean(signal, axis=1) # mean over the 32*32(FGS1) or 32(CH0) pixels\n        #print(mean_signal.shape)\n        cds_signal = (mean_signal[1::2] - mean_signal[0::2])\n        #print(cds_signal.shape)\n        binned = np.zeros((binned_dict[sensor]))\n        for j in range(cds_signal.shape[0] // binning):\n            binned[j] = cds_signal[j*binning:j*binning+binning].mean(axis=0)\n                   \n        if sensor == \"FGS1\":\n            binned = binned.reshape((binned.shape[0],1))\n        #print(binned.shape)    \n        feats.append(binned)\n        \n    return np.stack(feats)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''with open('pre_train2.pickle', 'wb') as f:\n    pickle.dump(pre_train, f)'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/test-daiding/pre_train.pickle', 'rb') as f:\n    pre_train = pickle.load(f)\nwith open('/kaggle/input/test-daiding/pre_train2.pickle', 'rb') as f:\n    pre_train2 = pickle.load(f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_train2[0][0][:20]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''with open('/kaggle/input/neurips-2024-ariel/pre_train.pickle', 'rb') as f:\n    pre_train = pickle.load(f)\nwith open('/kaggle/input/neurips-2024-ariel/pre_train2.pickle', 'rb') as f:\n    pre_train2 = pickle.load(f)'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def phase_detector2(signal):\n    phase1, phase2 = None, None\n    best_drop = 0\n    for i in range(50//2,150//2):\n        t1 = (signal[i:i+20//2].max() - signal[i:i+20//2].min())/signal[i:i+20//2].mean()\n        if t1 > best_drop:\n            phase1 = i+(20+5)//2\n            best_drop = t1\n    \n    best_drop = 0\n    for i in range(200//2,250//2):\n        t1 = (signal[i:i+20//2].max() - signal[i:i+20//2].min())/signal[i:i+20//2].mean()\n        if t1 > best_drop:\n            phase2 = i-5//2\n            best_drop = t1\n    \n    return phase1, phase2\n\ndef try_s(signal, p1, p2, deg, s):\n    out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n    x, y = out, signal[out].tolist()\n    x = x + list(range(p1,p2))\n\n    y = y + (signal[p1:p2] * (1 + s[0])).tolist()\n    z = np.polyfit(x, y, deg)\n    p = np.poly1d(z)\n    q = np.abs(p(x) - y).mean()\n\n    if s < 1e-4:\n        return q + 1e3\n\n    return q\n    \ndef calibrate_signal(signal):\n    p1,p2 = phase_detector2(signal)\n\n    best_deg, best_score = 1, 1e12\n    for deg in range(1, 6):\n        f = partial(try_s, signal, p1, p2, deg)\n        r = minimize(f, [0.001], method = 'Nelder-Mead')\n        s = r.x[0]\n\n        out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n        x, y = out, signal[out].tolist()\n        x = x + list(range(p1,p2))\n        y = y + (signal[p1:p2] * (1 + s)).tolist()\n    \n        z = np.polyfit(x, y, deg)\n        p = np.poly1d(z)\n        q = np.abs(p(x) - y).mean()\n        \n        if q < best_score:\n            best_score = q\n            best_deg = deg\n        \n        print(deg, q)\n            \n    z = np.polyfit(x, y, best_deg)\n    p = np.poly1d(z)\n\n    return s, x, y, p(x)\n\ndef calibrate_train(signal):\n    p1,p2 = phase_detector2(signal)\n    \n    best_deg, best_score = 1, 1e12\n    for deg in range(1, 4):\n        f = partial(try_s, signal, p1, p2, deg)\n        r = minimize(f, [0.0001], method = 'Nelder-Mead')\n        s = r.x[0]\n\n        out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n        x, y = out, signal[out].tolist()\n        x = x + list(range(p1,p2))\n        y = y + (signal[p1:p2] * (1 + s)).tolist()\n        #print(y)\n    \n        z = np.polyfit(x, y, deg)\n        p = np.poly1d(z)\n        q = np.abs(p(x) - y).mean()\n        \n        if q < best_score:\n            best_score = q\n            best_deg = deg\n            \n    z = np.polyfit(x, y, best_deg)\n    p = np.poly1d(z)\n    \n    return s, p(np.arange(signal.shape[0])), p1, p2\n'''\ntrain = pre_train.copy()\nall_s = []\nfor i in range(len(train_adc_info)):\n    tmp=[]\n    for j in range(0,283,10):\n        signal = train[i,:,j:j+10].mean(axis=1)\n        s, p, p1, p2 = calibrate_train(pre_train[i,:,j:j+10].mean(axis=1))\n        tmp.append(s)\n    all_s.append(tmp)\n    break\n#train_s=np.array(all_s)'''\n#copy answer 283 times because we predict mean value\n#train_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n# 设定总信号的均值\nglobal_mean = 0.0012150000000000025\n\n# 生成初始的局部均值信号，设定数量为30\nn_signals = 30\nlocal_means = np.random.normal(loc=global_mean, scale=0.00025, size=n_signals)\n\n# 保证局部均值的整体均值接近 global_mean\nadjusted_means = local_means - (local_means.mean() - global_mean)\n\n# 打印调整后的局部均值\nprint(adjusted_means)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def phase_detector(signal):\n    \n    MIN = np.argmin(signal[30:140])+30\n    signal1 = signal[:MIN ]\n    signal2 = signal[MIN :]\n\n    first_derivative1 = np.gradient(signal1)\n    first_derivative1 /= first_derivative1.max()\n    first_derivative2 = np.gradient(signal2)\n    first_derivative2 /= first_derivative2.max()\n\n    phase1 = np.argmin(first_derivative1)\n    phase2 = np.argmax(first_derivative2) + MIN\n\n    return phase1, phase2\n    \ndef objective(s):\n    \n    best_q = 1e10\n    for i in range(5) :\n        delta = 2\n        x = list(range(signal.shape[0]-delta*4))\n        y = signal[:p1-delta].tolist() + (signal[p1+delta:p2 - delta] * (1 + s - s**2)).tolist() + signal[p2+delta:].tolist()\n        \n        z = np.polyfit(x, y, deg=i)\n        p = np.poly1d(z)\n        q = np.abs(p(x) - y).mean()\n    \n    if q < best_q :\n        best_q = q\n    \n    return q\n\n'''\nall_s = []\nfor i in tqdm(range(len(train_adc_info))):\n    \n    signal = pre_train2[i,:,1:].mean(axis=1)\n    p1,p2 = phase_detector(signal)\n \n    r = minimize(\n                objective,\n                [0.0001],\n                method= 'Nelder-Mead'\n                  )\n    s = r.x[0]\n    all_s.append(s)\n    \ntrain_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''#abs(train_labels.mean(axis=1)-all_s).mean()\nr2score = r2_score(train_labels, train_s)\nprint(f\"# R2 score: {r2score:.3f}\")\nsigma_pred = mean_squared_error(train_labels, train_s, squared=False)\nprint(f\"# Root mean squared error: {sigma_pred:.6f}\")'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def f_read_and_preprocess(dataset, adc_info, planet_ids):\n    f_raw_train = np.full((len(planet_ids), 67500), np.nan, dtype=np.float32)\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        f_signal = pl.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/FGS1_signal.parquet')\n        #f_signal = pl.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/FGS1_signal.parquet').cast(pl.Float64).to_numpy()\n        #f_signal = ADC_convert(f_signal, adc_info.loc[int(planet_id)]['FGS1_adc_gain'], adc_info.loc[int(planet_id)]['FGS1_adc_offset'])\n        #dark = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/FGS1_calibration/dark.parquet').values.astype(np.float64)\n        #dead = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/FGS1_calibration/dead.parquet').values.astype(np.float64)\n        #flat = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/FGS1_calibration/flat.parquet').values.astype(np.float64)\n        #f_signal = mask_hot_dead(f_signal, dead, dark)\n        #dt_fgs1 = np.ones(len(f_signal))*0.1\n        #f_signal = clean_dark(f_signal, dead, dark, dt_fgs1)\n        #f_signal = correct_flat_field(flat,dead, f_signal)\n        mean_signal = f_signal.cast(pl.Int32).sum_horizontal().cast(pl.Float32).to_numpy() / 1024\n        #mean_signal = np.sort(f_signal.to_numpy(), axis=1)[:,-100:].mean(axis=1)\n        #mean_signal = f_signal.mean(axis=1)\n        net_signal = mean_signal[1::2] - mean_signal[0::2]\n        f_raw_train[i] = net_signal\n    return f_raw_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''f_raw_train = f_read_and_preprocess('train', train_adc_info, train_labels.index)\nwith open('f_raw_train.pickle', 'wb') as f:\n    pickle.dump(f_raw_train, f)'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''f_raw_train = f_read_and_preprocess('train', train_adc_info, train_labels.index)\nwith open('f_raw_train.pickle', 'wb') as f:\n    pickle.dump(f_raw_train, f)'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/d/zxl2022/neurips-2024-ariel/f_raw_train.pickle', 'rb') as f:\n    f_raw_train = pickle.load(f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def a_read_and_preprocess(dataset, adc_info, planet_ids):\n    a_raw_train = np.full((len(planet_ids), 5625), np.nan, dtype=np.float32)\n    #dt_airs = axis_info['AIRS-CH0-integration_time'].dropna().values\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        signal = pl.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/AIRS-CH0_signal.parquet')\n        #a_signal = pl.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/AIRS-CH0_signal.parquet').cast(pl.Float64).to_numpy()\n        #a_signal = ADC_convert(a_signal, adc_info.loc[int(planet_id)]['AIRS-CH0_adc_gain'], adc_info.loc[int(planet_id)]['AIRS-CH0_adc_offset'])\n        #dark = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/AIRS-CH0_calibration/dark.parquet').values.astype(np.float64)\n        #dead = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/AIRS-CH0_calibration/dead.parquet').values.astype(np.float64)\n        #flat = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/AIRS-CH0_calibration/flat.parquet').values.astype(np.float64)\n        #a_signal = mask_hot_dead(a_signal, dead, dark)\n        #a_signal = clean_dark(a_signal, dead, dark, dt_airs)\n        #a_signal = correct_flat_field(flat,dead, a_signal)\n        mean_signal = signal.cast(pl.Int32).sum_horizontal().cast(pl.Float32).to_numpy() / (32*356)\n        #mean_signal = np.sort(signal.to_numpy(), axis=1)[:,-2136:].mean(axis=1)\n        #mean_signal = a_signal.mean(axis=1)\n        net_signal = mean_signal[1::2] - mean_signal[0::2]\n        a_raw_train[i] = net_signal\n    return a_raw_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''a_raw_train = a_read_and_preprocess('train', train_adc_info, train_labels.index)\nwith open('a_raw_train.pickle', 'wb') as f:\n    pickle.dump(a_raw_train, f)'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/d/zxl2022/neurips-2024-ariel/a_raw_train.pickle', 'rb') as f:\n    a_raw_train = pickle.load(f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineering1(f_raw, a_raw, w0, window_size=50, step_size=25):\n\n    # Sliding window features\n    def sliding_window_features(data, window_size, step_size):\n        features = []\n        max_index = data.shape[1]\n        for start in range(0, max_index - window_size + 1, step_size):\n            end = start + window_size\n            window = data[:, start:end]\n            window_sorted = np.sort(window, axis=1)\n            features.append([\n                np.mean(window*w0, axis=1),\n            ])\n        if features:\n            return np.vstack(features).T\n        else:\n            return np.empty((data.shape[0], 0))\n    \n    f_sliding_features = sliding_window_features(f_raw, window_size, step_size)\n    a_sliding_features = sliding_window_features(a_raw, window_size, step_size)\n\n\n    df = pd.DataFrame({})\n\n\n    if f_sliding_features.size > 0:\n        f_sliding_df = pd.DataFrame(f_sliding_features, columns=[f'f_slide_{i}' for i in range(f_sliding_features.shape[1])])\n        df = pd.concat([df, f_sliding_df], axis=1)\n\n    if a_sliding_features.size > 0:\n        a_sliding_df = pd.DataFrame(a_sliding_features, columns=[f'a_slide_{i}' for i in range(a_sliding_features.shape[1])])\n        df = pd.concat([df, a_sliding_df], axis=1)\n    \n    return df\n\n'''ws = []\nfor _ in range(1000):\n    w0 = np.random.rand(50)\n    train1 = feature_engineering1(f_raw_train, a_raw_train, w0)\n    model = Ridge(alpha=1e-12)\n    oof_pred = cross_val_predict(model, train1, train_labels)\n    sigma_pred = mean_squared_error(train_labels, oof_pred, squared=False)\n    print(sigma_pred)\n    ws.append((sigma_pred, w0))'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''ws = sorted(ws)\nw0 = ws[0][1]\nw1 = ws[1][1]\nw2 = ws[2][1]\nw3 = ws[3][1]\nw4 = ws[4][1]\nw5 = ws[5][1]\nw6 = ws[6][1]\nw7 = ws[7][1]\nw8 = ws[8][1]\nw9 = ws[9][1]'''\n\ndef feature_engineering(f_raw, a_raw, window_size=50, step_size=25):\n    \n    #f_raw_sorted = np.sort(f_raw, axis=1)\n    #a_raw_sorted = np.sort(a_raw, axis=1)\n    \n    f_obscured = f_raw[:, 23500:44000].mean(axis=1)\n    f_unobscured = (f_raw[:, :20500].mean(axis=1) + f_raw[:, 47000:].mean(axis=1)) / 2\n    #f_obscured = f_raw_sorted[:, :20500].mean(axis=1)\n    #f_unobscured = f_raw_sorted[:, -41000:].mean(axis=1)\n    \n    f_relative_reduction = (f_unobscured - f_obscured) / f_unobscured\n    f_std_dev = f_raw.std(axis=1)\n    f_signal_to_noise = f_unobscured / f_std_dev \n\n    a_obscured = a_raw[:, 1958:3666].mean(axis=1)\n    a_unobscured = (a_raw[:, :1708].mean(axis=1) + a_raw[:, 3916:].mean(axis=1)) / 2\n    #a_obscured = a_raw_sorted[:, :1708].mean(axis=1)\n    #a_unobscured = a_raw_sorted[:, -3417:].mean(axis=1)\n    \n    a_relative_reduction = (a_unobscured - a_obscured) / a_unobscured\n    a_std_dev = a_raw.std(axis=1)\n    a_signal_to_noise = a_unobscured / a_std_dev\n\n    f_variance = f_raw.var(axis=1)\n    a_variance = a_raw.var(axis=1)\n    \n    f_obscured_variance = f_raw[:, 23500:44000].var(axis=1)\n    f_unobscured_variance = (f_raw[:, :20500].var(axis=1) + f_raw[:, 47000:].var(axis=1)) / 2\n    f_variance_relative_reduction = (f_unobscured_variance - f_obscured_variance) / f_unobscured_variance\n    a_obscured_variance = a_raw[:, 1958:3666].var(axis=1)\n    a_unobscured_variance = (a_raw[:, :1708].var(axis=1) + a_raw[:, 3916:].var(axis=1)) / 2\n    a_variance_relative_reduction = (a_unobscured_variance - a_obscured_variance) / a_unobscured_variance\n    f_obscured_ptp = np.ptp(f_raw[:, 23500:44000], axis=1)\n    f_unobscured_ptp = (np.ptp(f_raw[:, :20500], axis=1) + np.ptp(f_raw[:, 47000:], axis=1)) / 2\n    f_ptp_relative_reduction = (f_unobscured_ptp - f_obscured_ptp) / f_unobscured_ptp\n    a_obscured_ptp = np.ptp(a_raw[:, 1958:3666], axis=1)\n    a_unobscured_ptp = (np.ptp(a_raw[:, :1708], axis=1) + np.ptp(a_raw[:, 3916:], axis=1)) / 2\n    a_ptp_relative_reduction = (a_unobscured_ptp - a_obscured_ptp) / a_unobscured_ptp\n    \n    f_skewness = pd.DataFrame(f_raw).skew(axis=1).values\n    a_skewness = pd.DataFrame(a_raw).skew(axis=1).values\n\n    f_kurtosis = pd.DataFrame(f_raw).kurtosis(axis=1).values\n    a_kurtosis = pd.DataFrame(a_raw).kurtosis(axis=1).values\n    \n    f_half_obscured1 = f_raw[:, 20500:23500].mean(axis=1)\n    f_half_obscured2 = f_raw[:, 44000:47000].mean(axis=1)\n    #f_half_obscured1 = f_raw_sorted[:, 20500:26500].mean(axis=1)\n    #f_half_obscured2 = f_raw_sorted[:, -47000:-41000].mean(axis=1)\n    f_half_reduction1 = (f_unobscured - f_half_obscured1) / f_unobscured\n    f_half_reduction2 = (f_unobscured - f_half_obscured2) / f_unobscured\n\n    a_half_obscured1 = a_raw[:, 1708:1958].mean(axis=1)\n    a_half_obscured2 = a_raw[:, 3666:3916].mean(axis=1)\n    #a_half_obscured1 = a_raw_sorted[:, 1708:2208].mean(axis=1)\n    #a_half_obscured2 = a_raw_sorted[:, -3917:-3417].mean(axis=1)\n    a_half_reduction1 = (a_unobscured - a_half_obscured1) / a_unobscured\n    a_half_reduction2 = (a_unobscured - a_half_obscured2) / a_unobscured\n\n    # Sliding window features\n    def sliding_window_features(data, window_size, step_size):\n        features = []\n        max_index = data.shape[1]\n        for start in range(0, max_index - window_size + 1, step_size):\n            end = start + window_size\n            window = data[:, start:end]\n            #window_sorted = np.sort(window, axis=1)\n            features.append([\n                np.mean(window, axis=1),\n                np.std(window, axis=1),\n                #np.var(window, axis=1),\n                np.median(window, axis=1),\n                #np.min(window, axis=1),\n                #np.max(window, axis=1),\n                #np.percentile(window, 95, axis=1),\n                #np.percentile(window, 75, axis=1),\n                #np.percentile(window, 55, axis=1),\n                #np.percentile(window, 35, axis=1),\n                #np.percentile(window, 15, axis=1),\n                np.ptp(window, axis=1)/np.max(window, axis=1)\n                #np.mean(window*w0, axis=1),\n                #scipy.stats.skew(window, axis=1, bias=True),\n                #scipy.stats.kurtosis(window, axis=1, bias=True)\n            ])\n        if features:\n            return np.vstack(features).T\n        else:\n            return np.empty((data.shape[0], 0))\n    \n    f_sliding_features = sliding_window_features(f_raw, window_size, step_size)\n    a_sliding_features = sliding_window_features(a_raw, window_size, step_size)\n\n\n    #print(f'f_sliding_features.shape: {f_sliding_features.shape}')\n    #print(f'a_sliding_features.shape: {a_sliding_features.shape}')\n\n\n    df = pd.DataFrame({\n        'f_relative_reduction': f_relative_reduction,\n        'f_signal_to_noise': f_signal_to_noise,\n        'f_variance': f_variance,\n        'f_skewness': f_skewness,\n        'f_kurtosis': f_kurtosis,\n        'a_relative_reduction': a_relative_reduction,\n        'a_signal_to_noise': a_signal_to_noise,\n        'a_variance': a_variance,\n        'a_skewness': a_skewness,\n        'a_kurtosis': a_kurtosis,\n        'f_half_reduction1': f_half_reduction1,\n        'f_half_reduction2': f_half_reduction2,\n        'a_half_reduction1': a_half_reduction1,\n        'a_half_reduction2': a_half_reduction2,\n        'f_variance_relative_reduction': f_variance_relative_reduction,\n        'a_variance_relative_reduction': a_variance_relative_reduction,\n        'f_ptp_relative_reduction' : f_ptp_relative_reduction,\n        'a_ptp_relative_reduction' : a_ptp_relative_reduction\n    })\n\n\n    if f_sliding_features.size > 0:\n        f_sliding_df = pd.DataFrame(f_sliding_features, columns=[f'f_slide_{i}' for i in range(f_sliding_features.shape[1])])\n        df = pd.concat([df, f_sliding_df], axis=1)\n\n    if a_sliding_features.size > 0:\n        a_sliding_df = pd.DataFrame(a_sliding_features, columns=[f'a_slide_{i}' for i in range(a_sliding_features.shape[1])])\n        df = pd.concat([df, a_sliding_df], axis=1)\n    \n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = feature_engineering(f_raw_train*0.3, a_raw_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc = 0.997-0.000089\nwm = np.array([0.3,1.0])\n'''for _ in range(3000):\n    w = np.sort(np.random.rand(2))\n    train = feature_engineering(f_raw_train*w[0], a_raw_train*w[1])\n    #{'sag', 'lsqr', 'svd', 'auto', 'cholesky', 'lbfgs', 'saga', 'sparse_cg'}\n    model = Ridge(alpha=1e-12)\n    #{'cosine', 'additive_chi2', 'rbf', 'poly', 'precomputed', 'linear', 'chi2', 'sigmoid', 'polynomial', 'laplacian'}\n    #model1 = KernelRidge(alpha=0.0005, kernel='additive_chi2')\n\n    oof_pred = cross_val_predict(model, train, train_labels)\n    r2score = r2_score(train_labels, oof_pred)\n\n    print(f\"# R2 score: {r2score:.3f}\")\n    sigma_pred = mean_squared_error(train_labels, oof_pred, squared=False)\n    print(f\"# Root mean squared error: {sigma_pred:.6f}\")\n    if r2score-sigma_pred > sc:\n        sc = r2score-sigma_pred\n        wm = w'''\ntrain = feature_engineering(f_raw_train*wm[0], a_raw_train*wm[1])\nwm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(a_raw_train[0])\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### {'sag', 'lsqr', 'svd', 'auto', 'cholesky', 'lbfgs', 'saga', 'sparse_cg'}\nmodel = Ridge(alpha=1e-12)\n#{'cosine', 'additive_chi2', 'rbf', 'poly', 'precomputed', 'linear', 'chi2', 'sigmoid', 'polynomial', 'laplacian'}\n#model1 = KernelRidge(alpha=0.0005, kernel='additive_chi2')\n\noof_pred = cross_val_predict(model, train, train_labels)\n\n'''oof_pred += (np.array(all_s) - oof_pred.mean(axis=1)).reshape(-1,1)*0.5\n\noof_pred += (oof_pred.mean(axis=1).reshape(-1,1)-oof_pred)*0.25\n\nr2score = r2_score(train_labels, oof_pred)\n\nprint(f\"# R2 score: {r2score:.3f}\")\nsigma_pred = mean_squared_error(train_labels, oof_pred, squared=False)\n\nprint(f\"# Root mean squared error: {sigma_pred:.6f}\")'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sigma_pred = train_labels-oof_pred\nsigma_pred=sigma_pred.abs()\nsigma_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ParticipantVisibleError(Exception):\n    pass\n\ndef competition_score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        naive_mean: float,\n        naive_sigma: float,\n        sigma_true: float,\n        row_id_column_name='planet_id',\n    ) -> float:\n\n    del solution[row_id_column_name]\n    del submission[row_id_column_name]\n\n    if submission.min().min() < 0:\n        raise ParticipantVisibleError('Negative values in the submission')\n    for col in submission.columns:\n        if not pd.api.types.is_numeric_dtype(submission[col]):\n            raise ParticipantVisibleError(f'Submission column {col} must be a number')\n\n    n_wavelengths = len(solution.columns)\n    if len(submission.columns) != n_wavelengths*2:\n        raise ParticipantVisibleError('Wrong number of columns in the submission')\n\n    y_pred = submission.iloc[:, :n_wavelengths].values\n    # Set a non-zero minimum sigma pred to prevent division by zero errors.\n    sigma_pred = np.clip(submission.iloc[:, n_wavelengths:].values, a_min=10**-15, a_max=None)\n    y_true = solution.values\n\n    GLL_pred = np.sum(scipy.stats.norm.logpdf(y_true, loc=y_pred, scale=sigma_pred))\n    GLL_true = np.sum(scipy.stats.norm.logpdf(y_true, loc=y_true, scale=sigma_true * np.ones_like(y_true)))\n    GLL_mean = np.sum(scipy.stats.norm.logpdf(y_true, loc=naive_mean * np.ones_like(y_true), scale=naive_sigma * np.ones_like(y_true)))\n\n    submit_score = (GLL_pred - GLL_mean)/(GLL_true - GLL_mean)\n    return float(np.clip(submit_score, 0.0, 1.0))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''def postprocessing(pred_array, index, sigma_pred):\n    return pd.concat([pd.DataFrame(pred_array.clip(0, None), index=index, columns=wavelengths.columns),\n                      pd.DataFrame(sigma_pred, index=index, columns=[f\"sigma_{i}\" for i in range(1, 284)])],\n                     axis=1)'''\n\nsigma_pred=pd.DataFrame(sigma_pred.values,columns=[f\"sigma_{i}\" for i in range(1, 284)])\n\ndef postprocessing(pred_array, index, sigma_pred):\n    return pd.concat([pd.DataFrame(pred_array.clip(0, None), index=index, columns=wavelengths.columns),\n                      pd.DataFrame(sigma_pred.values.clip(0, None), index=index, columns=[f\"sigma_{i}\" for i in range(1, 284)])],\n                     axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sigma_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df = postprocessing(oof_pred, train_adc_info.index, sigma_pred)\ndisplay(oof_df)\n\ngll_score = competition_score(train_labels.copy().reset_index(),\n                              oof_df.copy().reset_index(),\n                              naive_mean=train_labels.values.mean(),\n                              naive_sigma=train_labels.values.std(),\n                              sigma_true=0.000001)\nprint(f\"# Estimated competition score: {gll_score:.3f}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train, train_labels)\ntrain_pred = model.predict(train)\nwith open('model.pickle', 'wb') as f:\n    pickle.dump(model, f)\nwith open('sigma_pred.pickle', 'wb') as f:\n    pickle.dump(sigma_pred, f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''def postprocessing(pred_array, index, sigma_pred):\n    return pd.concat([pd.DataFrame(pred_array.clip(0, None), index=index, columns=wavelengths.columns),\n                      pd.DataFrame(sigma_pred.clip(0.000155, None), index=index, columns=[f\"sigma_{i}\" for i in range(1, 284)])],\n                     axis=1)'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/test_adc_info.csv', index_col='planet_id')\nf_raw_test = f_read_and_preprocess('test', test_adc_info, test_adc_info.index)\na_raw_test = a_read_and_preprocess('test', test_adc_info, test_adc_info.index)\ntest = feature_engineering(f_raw_test*wm[0], a_raw_test*wm[1])\nprint(test)\nprint(f\"Number of features:{test.shape[1]}\")\n\n# Load the model\nwith open('model.pickle', 'rb') as f:\n    model = pickle.load(f)\nwith open('sigma_pred.pickle', 'rb') as f:\n    sigma_pred = pickle.load(f)\n    \n# Predict\npre_test = np.concatenate([preproc('test', test_adc_info, \"FGS1\", 30*12), preproc('test', test_adc_info, \"AIRS-CH0\", 30)], axis=2)\npre_test2 = np.concatenate([preproc2('test', test_adc_info, \"FGS1\", 30*12), preproc2('test', test_adc_info, \"AIRS-CH0\", 30)], axis=2)\ntest_pred = model.predict(test)\n\ntest1 = pre_test.copy()\ntest2 = pre_test2.copy()\n\nsigma_pred2 = []\n\nfor i in range(len(test_adc_info)):\n    if test_adc_info.iloc[i]['star'] == 1:\n        signal = pre_test[i,:,1:].mean(axis=1)\n        p1,p2 = phase_detector(signal)\n \n        r = minimize(objective, [0.0001], method= 'Nelder-Mead')\n        s = r.x[0]\n        \n        n_signals = 283\n        local_means = np.random.normal(loc=s, scale=0.00000405, size=n_signals)\n\n        # 保证局部均值的整体均值接近 global_mean\n        adjusted_means = local_means - (local_means.mean() - s)\n        \n        signal2 = test2[i,:,1:].mean(axis=1)\n        s2, p, p1, p2 = calibrate_train(pre_test2[i,:,1:].mean(axis=1))\n        #test_pred[i] += (0.0302*s2 + 0.9698*adjusted_means - test_pred[i].mean())*0.5\n        #original 0.0302 0.9698 best 0.15 0.87\n        test_pred[i] += (0.16*s2 + 0.84*adjusted_means - test_pred[i].mean())*0.5\n        \n        sorted_preds = np.sort(test_pred[i])  \n        n = len(sorted_preds)  \n\n        # 计算四分位数  \n        Q1 = np.percentile(sorted_preds, 25)\n        Q2=np.percentile(sorted_preds,50)\n        Q3 = np.percentile(sorted_preds, 77) #best 77\n        Q4=np.percentile(sorted_preds,90)\n        \n        test_pred[i] += (Q3-test_pred[i])*0.25\n        #test_pred[i] += (test_pred[i].mean()-test_pred[i])*0.25\n        #test_pred[i] = 0.0302*s2 + 0.9698*s\n        #test_pred[i] = 0.1*s2 + 0.9*s\n\n    else:\n        signal = pre_test[i,:,1:].mean(axis=1)\n        p1,p2 = phase_detector(signal)\n \n        r = minimize(objective, [0.0001], method= 'Nelder-Mead')\n        s = r.x[0]\n        signal2 = test2[i,:,1:].mean(axis=1)\n\n        n_signals = 283\n        local_means = np.random.normal(loc=s, scale=0.00000405, size=n_signals)\n\n        # 保证局部均值的整体均值接近 global_mean\n        adjusted_means = local_means - (local_means.mean() - s)\n        \n        signal2 = test2[i,:,1:].mean(axis=1)\n        s2, p, p1, p2 = calibrate_train(pre_test2[i,:,1:].mean(axis=1))\n        \n        #s2, p, p1, p2 = calibrate_train(pre_test2[i,:,1:].mean(axis=1))\n        #test_pred[i] += s-test_pred[i].mean()\n        #test_pred[i] += (test_pred[i].mean()-test_pred[i])*1.5\n        #original 0.0302 0.9698 best 0.15 0.85\n        test_pred[i] = 0.16*s2 + 0.84*s\n        #test_pred[i] = 0.1*s2 + 0.9*s\n    \n    if test_adc_info.iloc[i]['star'] == 0:\n        sigma_pred2.append([0.000135])\n    elif test_adc_info.iloc[i]['star'] == 1:\n        sigma_pred2.append([0.0001])\n    elif test_adc_info.iloc[i]['star'] == 2:\n        sigma_pred2.append([0.000185])\n    else:\n        sigma_pred2.append([0.000145])\n    \n\n'''train_labels_values = train_labels.values\nsigma_pred = []\nfor i in range(len(test_pred)):\n    distances = []\n    for j in range(len(train_labels_values)):\n        distance = np.sum((train_labels_values[j]-test_pred[i])**2)\n        distances.append((distance, j))\n    distances = sorted(distances)\n    tmp = []\n    for ii in range(30):\n        tmp.append((train_labels_values[distances[ii][1]]-test_pred[i])**2)\n    #test_pred[i] = np.mean(np.array(tmp), axis=0)\n    sigma_pred.append(np.mean(np.array(tmp), axis=0)**0.5)\nsigma_pred = np.array(sigma_pred)'''\n\n# Package into submission file\n#sub_df = postprocessing(test_pred,\n#                        test_adc_info.index,\n#                        sigma_pred=np.tile(np.where(test_adc_info[['star']] <= 1, 0.000135, 0.000145), (1, 283)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sigmas=np.load('/kaggle/input/moblienet-v2/purple_hat_oof_sigmas (1).npy')\n'''\nimport numpy as np\nimport os\n\n# 文件夹路径\nfolder_path = '/kaggle/input/purple-hat-oof-sigmas0-3'\n\n# 初始化一个列表来存储所有读取的数组\nall_arrays = []\n\n# 遍历文件夹中的所有文件\nfor filename in os.listdir(folder_path):\n    if filename.endswith('.npy'):\n        file_path = os.path.join(folder_path, filename)\n        # 读取npy文件并添加到列表中\n        array = np.load(file_path)\n        all_arrays.append(array)\n\n# 计算所有数组的平均值\nsigmas = np.mean(all_arrays, axis=0)\n\n#print(\"平均值数组形状:\", average_array.shape)\n'''\n#vit_train_sigmas=sigmas*0.17 \n#vit_train_sigmas=sigmas*0.2301 best\nvit_train_sigmas=sigmas*0.2301\n#vit_train_sigmas=sigmas*0.254\nfrom sklearn.linear_model import Ridge\nfrom sklearn.model_selection import GridSearchCV\n\nall_s_train=train_labels\nreal_train=vit_train_sigmas\n\n# 构建Ridge回归模型\nsigmas_model = Ridge()\n\n# 定义超参数网格\nparam_grid = {\n    'alpha': [1e-012,1e-06,0.01, 0.1, 1, 10, 100],  # 尝试不同的正则化强度\n}\n\n# 使用网格搜索寻找最佳超参数\ngrid_search = GridSearchCV(sigmas_model, param_grid, cv=5, scoring='neg_mean_squared_error')\ngrid_search.fit(all_s_train, real_train)\n\n# 获取最佳模型\nbest_sigmas_model = grid_search.best_estimator_\nprint(f'最佳超参数: {grid_search.best_params_}')\nprint(f'最佳得分: {grid_search.best_score_}')\n\nbest_sigmas_model.fit(all_s_train, real_train)\n\nfrom sklearn.metrics import mean_squared_error\n\n# 预测结果\npredictions = best_sigmas_model.predict(all_s_train)\n\n# 计算均方误差\nmse = mean_squared_error(real_train, predictions)\nrmse = np.sqrt(mse)\nprint(f'训练集上的RMSE: {rmse}')\n\nsigma_fake=np.abs(real_train-predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def postprocessing(pred_array, index, sigma_pred):\n    return pd.concat([pd.DataFrame(pred_array.clip(0, None), index=index, columns=wavelengths.columns),\n                      pd.DataFrame(sigma_pred, index=index, columns=[f\"sigma_{i}\" for i in range(1, 284)])],\n                     axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 预测结果\n#np.concatenate((train_labels,sigma_pred),axis=1)\n#predictions = best_sigmas_model.predict(test_pred)\npredictions = best_sigmas_model.predict(test_pred)\n\n\nfor i in range(predictions.shape[0]):\n    for j in range(predictions.shape[1]):\n        if predictions[i][j] < sigma_pred2[i][0]-0.00004:\n            predictions[i][j]=sigma_pred2[i][0]-0.00004\n        elif predictions[i][j] >= sigma_pred2[i][0]+0.000038:\n            predictions[i][j]=sigma_pred2[i][0]+0.000038\n\n#tmp_best\n'''\nfor i in range(predictions.shape[0]):\n    for j in range(predictions.shape[1]):\n        if predictions[i][j] < sigma_pred2[i][0]-0.00004:\n            predictions[i][j]=sigma_pred2[i][0]-0.00004\n        elif predictions[i][j] >= sigma_pred2[i][0]+0.000038:\n            predictions[i][j]=sigma_pred2[i][0]+0.000038\n'''\n'''\nfor i in range(predictions.shape[0]):\n    for j in range(predictions.shape[1]):\n        if predictions[i][j] < sigma_pred2[i][0]-0.00004:\n            predictions[i][j]=sigma_pred2[i][0]-0.00004\n        elif predictions[i][j] >= sigma_pred2[i][0]+0.00004:\n            predictions[i][j]=sigma_pred2[i][0]+0.00004\n'''\nsub_df = postprocessing(test_pred, test_adc_info.index, predictions)\ndisplay(sub_df)\nsub_df.to_csv('submission.csv')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# 绘制折线图\nplt.figure(figsize=(10, 6))  # 设置图形大小\nplt.plot(predictions[0], label='Predicted Sigma', color='b', marker='o', linestyle='-')\n\n# 添加标题和标签\nplt.title('Predicted Sigma over Samples', fontsize=16)\nplt.xlabel('Sample Index', fontsize=14)\nplt.ylabel('Predicted Sigma Value', fontsize=14)\n\n# 显示图例\nplt.legend()\n\n# 显示图表\nplt.grid(True)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}