{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":9604843,"sourceType":"datasetVersion","datasetId":5860111},{"sourceId":9715129,"sourceType":"datasetVersion","datasetId":5943223},{"sourceId":138934,"sourceType":"modelInstanceVersion","modelInstanceId":90108,"modelId":114332},{"sourceId":139905,"sourceType":"modelInstanceVersion","modelInstanceId":90108,"modelId":114332},{"sourceId":140911,"sourceType":"modelInstanceVersion","modelInstanceId":90108,"modelId":114332},{"sourceId":142811,"sourceType":"modelInstanceVersion","modelInstanceId":90108,"modelId":114332},{"sourceId":143895,"sourceType":"modelInstanceVersion","modelInstanceId":90108,"modelId":114332},{"sourceId":146624,"sourceType":"modelInstanceVersion","modelInstanceId":90108,"modelId":114332},{"sourceId":149816,"sourceType":"modelInstanceVersion","modelInstanceId":90108,"modelId":114332},{"sourceId":152697,"sourceType":"modelInstanceVersion","modelInstanceId":129684,"modelId":152544}],"dockerImageVersionId":30746,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction \n\nThis notebook used 1D CNN + unsupervised polynomial fitting to predict the spectra mean and uncertainty estimation . Then used 2D-CNN to predict the residuals. Credits to the following notebooks:\n\n* Host starter notebook https://www.kaggle.com/code/gordonyip/host-starter-solution\n* @AmbrosM idea https://www.kaggle.com/competitions/ariel-data-challenge-2024/discussion/530152#2969648\n* ariel_only_correlation by Sergei Fironov, https://www.kaggle.com/code/sergeifironov/ariel-only-correlation\n* Ariel Data Challenge 2024 by expert qianc https://www.kaggle.com/code/xiaocao123/ariel-data-challenge-2024\n","metadata":{}},{"cell_type":"markdown","source":"# Preprocess the data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\nimport scipy.stats\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import r2_score, mean_squared_error\nimport itertools\nfrom scipy.optimize import minimize\nfrom functools import partial\nimport random, os\nfrom astropy.stats import sigma_clip\nimport pickle","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-01T17:43:16.382777Z","iopub.execute_input":"2024-11-01T17:43:16.383197Z","iopub.status.idle":"2024-11-01T17:43:18.434438Z","shell.execute_reply.started":"2024-11-01T17:43:16.383153Z","shell.execute_reply":"2024-11-01T17:43:18.433322Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/test_adc_info.csv',\n index_col='planet_id')\naxis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')\ntrain_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_adc_info.csv',\n index_col='planet_id')\ntrain_labels = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_labels.csv',\n index_col='planet_id')\nwave_length = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')\n","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:18.437042Z","iopub.execute_input":"2024-11-01T17:43:18.437548Z","iopub.status.idle":"2024-11-01T17:43:18.805951Z","shell.execute_reply.started":"2024-11-01T17:43:18.437517Z","shell.execute_reply":"2024-11-01T17:43:18.804612Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DEBUG=False\ndef apply_linear_corr(linear_corr,clean_signal):\n linear_corr = np.flip(linear_corr, axis=0)\n for x, y in itertools.product(\n range(clean_signal.shape[1]), range(clean_signal.shape[2])\n ):\n poli = np.poly1d(linear_corr[:, x, y])\n clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n return clean_signal\n\ndef clean_dark(signal, dark, dt):\n dark = np.tile(dark, (signal.shape[0], 1, 1))\n if(DEBUG):\n print(\"clean dark\")\n print(dark.shape)\n print(dark)\n print(\"signal before clean dark\")\n print(signal)\n signal -= dark* dt[:, np.newaxis, np.newaxis]\n if(DEBUG):\n print(\"signale after clean dark\")\n print(signal)\n return signal\n\ndef preproc(dataset, adc_info, sensor, binning = 15):\n cut_inf, cut_sup = 39, 321\n sensor_sizes_dict = {\"AIRS-CH0\":[[11250, 32, 356], [1, 32, cut_sup-cut_inf]], \"FGS1\":[[135000, 32, 32], [1, 32, 32]]}\n binned_dict = {\"AIRS-CH0\":[11250 // binning // 2, 282], \"FGS1\":[135000 // binning // 2]}\n linear_corr_dict = {\"AIRS-CH0\":(6, 32, 356), \"FGS1\":(6, 32, 32)}\n planet_ids = adc_info.index\n \n feats = []\n for i, planet_id in tqdm(list(enumerate(planet_ids))):\n signal = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/{sensor}_signal.parquet').to_numpy()\n dark_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dark.parquet', engine='pyarrow').to_numpy()\n dead_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dead.parquet', engine='pyarrow').to_numpy()\n flat_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/flat.parquet', engine='pyarrow').to_numpy()\n linear_corr = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/linear_corr.parquet').values.astype(np.float64).reshape(linear_corr_dict[sensor])\n read_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/read.parquet', engine='pyarrow').to_numpy()\n signal = signal.reshape(sensor_sizes_dict[sensor][0]) \n if(DEBUG):\n print(signal.shape)\n print(signal)\n gain = adc_info[f'{sensor}_adc_gain'].values[i]\n offset = adc_info[f'{sensor}_adc_offset'].values[i]\n signal = signal / gain + offset\n # remove read noise\n signal = signal - read_frame\n if(DEBUG):\n print(signal)\n \n print(\"<<<< dark_frame>>>>>\")\n print(dark_frame.shape)\n print(dark_frame)\n hot = sigma_clip(\n dark_frame, sigma=5, maxiters=5\n ).mask\n if(DEBUG):\n print(hot)\n \n if sensor != \"FGS1\":\n signal = signal[:, :, cut_inf:cut_sup] #11250 * 32 * 282\n #dt = axis_info['AIRS-CH0-integration_time'].dropna().values\n dt = np.ones(len(signal))*0.1 \n dt[1::2] += 4.5 #@bilzard idea\n linear_corr = linear_corr[:, :, cut_inf:cut_sup]\n dark_frame = dark_frame[:, cut_inf:cut_sup]\n dead_frame = dead_frame[:, cut_inf:cut_sup]\n flat_frame = flat_frame[:, cut_inf:cut_sup]\n read_frame = read_frame[:,cut_inf:cut_sup]\n hot = hot[:, cut_inf:cut_sup]\n else:\n dt = np.ones(len(signal))*0.1\n dt[1::2] += 0.1\n if(DEBUG):\n print(\"<<<<< dt <<<<\")\n print(dt.shape)\n print(dt)\n signal = signal.clip(0) #@graySnow idea\n if(DEBUG): \n print(\"signal after clip 0\")\n print(signal)\n linear_corr_signal = apply_linear_corr(linear_corr, signal)\n signal = clean_dark(linear_corr_signal, dark_frame, dt)\n \n flat = flat_frame.reshape(sensor_sizes_dict[sensor][1])\n if(DEBUG):\n print(\"flat\")\n print(flat)\n print(\"count nan = \",np.count_nonzero(np.isnan(flat)))\n flat[dead_frame.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n flat[hot.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n if(DEBUG):\n print(\"<<<<<>>>feats<<<<<<\")\n print(binned.shape)\n print(len(feats[0]))\n print(feats)\n \n return np.stack(feats)\n \npre_train = np.concatenate([preproc('test', test_adc_info, \"FGS1\", 30*12), preproc('test', test_adc_info, \"AIRS-CH0\", 30)], axis=2)","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:18.807538Z","iopub.execute_input":"2024-11-01T17:43:18.807905Z","iopub.status.idle":"2024-11-01T17:43:31.554861Z","shell.execute_reply.started":"2024-11-01T17:43:18.807876Z","shell.execute_reply":"2024-11-01T17:43:31.553787Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# normalize signal\ndef data_norm(data1):\n data_min = data1.min()\n data_max = data1.max()\n #print(data_min,data_max)\n data_abs_max = np.max([abs(data_min), abs(data_max)])\n #print(data_abs_max)\n data1_norm = (data1-data_min)/(data_max - data_min)\n return data1_norm","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:31.55628Z","iopub.execute_input":"2024-11-01T17:43:31.55663Z","iopub.status.idle":"2024-11-01T17:43:31.562932Z","shell.execute_reply.started":"2024-11-01T17:43:31.5566Z","shell.execute_reply":"2024-11-01T17:43:31.561782Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Fit polynomials for each sample","metadata":{}},{"cell_type":"code","source":"\ndef phase_detector(signal):\n MIN = np.argmin(signal[30:140])+30\n signal1 = signal[:MIN ]\n signal2 = signal[MIN :]\n\n first_derivative1 = np.gradient(signal1)\n first_derivative1 /= first_derivative1.max()\n first_derivative2 = np.gradient(signal2)\n first_derivative2 /= first_derivative2.max()\n\n phase1 = np.argmin(first_derivative1)\n phase2 = np.argmax(first_derivative2) + MIN\n\n print(\"phase 1 and phase 2: \", phase1, phase2)\n return phase1, phase2\n\ndef objective(s):\n \n best_q = 1e10\n for i in range(4) :\n delta = 2\n x = list(range(signal.shape[0]-delta*4))\n y = signal[:p1-delta].tolist() + (signal[p1+delta:p2 - delta] * (1 + s)).tolist() + signal[p2+delta:].tolist()\n \n z = np.polyfit(x, y, deg=i)\n p = np.poly1d(z)\n q = np.abs(p(x) - y).mean()\n \n if q < best_q :\n best_q = q\n \n return q\n\n \ndef calibrate_signal(signal):\n p1,p2 = phase_detector(signal)\n\n best_deg, best_score = 1, 1e12\n for deg in range(1, 6):\n f = partial(try_s, signal, p1, p2, deg)\n r = minimize(f, [0.001], method = 'Nelder-Mead')\n s = r.x[0]\n\n out = list(range(p1-30)) + list(range(p2+30,signal.shape[0]))\n x, y = out, signal[out].tolist()\n if(DEBUG and deg < 2):\n print(\"x.shape, y.shape\", len(x), len(y))\n \n x = x + list(range(p1,p2))\n y = y + (signal[p1:p2] * (1 + s)).tolist()\n if(DEBUG and deg < 2):\n print(\"x.shape, y.shape\", len(x), len(y))\n \n z = np.polyfit(x, y, deg)\n p = np.poly1d(z)\n q = np.abs(p(x) - y).mean()\n \n if q < best_score:\n best_score = q\n best_deg = deg\n \n print(\"best deg and best_score\",best_deg, best_score)\n \n z = np.polyfit(x, y, best_deg)\n p = np.poly1d(z)\n\n return s, x, y, p(x)\n\ntrain = pre_train.copy()\nprint(train.shape)\nall_s = []\nall_p = []\nall_q = []\ndelta = 2\nfor i in tqdm(range(len(test_adc_info))):\n \n signal = pre_train[i,:,1:].mean(axis=1)\n p1,p2 = phase_detector(signal)\n \n all_p.append(p1)\n all_p.append(p2)\n r = minimize(\n objective,\n [0.00001],\n method= 'Nelder-Mead'\n )\n s = r.x[0]\n all_s.append(s)\n x = list(range(signal.shape[0]-delta*4))\n y = signal[:p1-delta].tolist() + (signal[p1+delta:p2 - delta] * (1 + s)).tolist() + signal[p2+delta:].tolist()\n \n z = np.polyfit(x, y, deg=4)\n p = np.poly1d(z)\n q = np.abs(p(x) - y).mean()\n all_q.append(q)\n \nall_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283)) \n\ntrain_s = all_s\ntrain_p = np.array(all_p).reshape((len(test_adc_info),2))\ntrain_q = np.array(all_q).reshape((len(test_adc_info),1))\n\ntrain_sigma = np.ones_like(train_s) * 0.000145","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:31.566932Z","iopub.execute_input":"2024-11-01T17:43:31.567708Z","iopub.status.idle":"2024-11-01T17:43:31.666157Z","shell.execute_reply.started":"2024-11-01T17:43:31.567663Z","shell.execute_reply":"2024-11-01T17:43:31.665095Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# make corretion on mean\n\ndef suppress_mean(targets, mean) : \n res = targets - np.repeat(mean.reshape((mean.shape[0], 1)), repeats = targets.shape[1], axis = 1)\n return res\n\nmake_correction_file = False\nif make_correction_file:\n train_targets = train_labels.to_numpy()\n pred_mean = np.clip(train_s.mean(axis=1),1e-10,None)\n print(pred_mean.shape)\n train_targets_shift = suppress_mean(train_targets, pred_mean)\n train_targets_shift_pct = train_targets_shift/pred_mean[:,np.newaxis]\n correction = train_targets_shift_pct.mean(axis=0)\n correction = correction.clip(-0.03,0.06)\nelse:\n correction = pickle.load(open(\"/kaggle/input/airel2024-correction/ariel_correction.pkl\", \"rb\" ) )\nprint(correction.shape)","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:31.667538Z","iopub.execute_input":"2024-11-01T17:43:31.667867Z","iopub.status.idle":"2024-11-01T17:43:31.687882Z","shell.execute_reply.started":"2024-11-01T17:43:31.66784Z","shell.execute_reply":"2024-11-01T17:43:31.686827Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1D CNN for target mean\n\nThese 1 D CNN models were pre-trained using the model structure given in Host's notebook, but changed SGD to Adam with 0.0001 learning rate. Each model was trained by a different random seed.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import load_model\n\ndef simpleScale(dataIn,dataMin,dataMax):\n dataOut = (dataIn - dataMin)/(dataMax - dataMin)\n return dataOut\n\ndef unstandardizing (data, min_train_valid, max_train_valid) : \n return data * (max_train_valid - min_train_valid) + min_train_valid","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:31.689168Z","iopub.execute_input":"2024-11-01T17:43:31.689534Z","iopub.status.idle":"2024-11-01T17:43:45.912001Z","shell.execute_reply.started":"2024-11-01T17:43:31.689506Z","shell.execute_reply":"2024-11-01T17:43:45.910889Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset = pre_train\n\nwc_mean = dataset[:,:,1:283].mean(axis=1).mean(axis=1)\nprint(\"wc_mean shape= \", wc_mean.shape)\n\nwhite_curve = dataset[:,:,1:283].sum(axis=2)/ wc_mean[:, np.newaxis] # normalized by each planet mean","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:45.91352Z","iopub.execute_input":"2024-11-01T17:43:45.914351Z","iopub.status.idle":"2024-11-01T17:43:45.92264Z","shell.execute_reply.started":"2024-11-01T17:43:45.914311Z","shell.execute_reply":"2024-11-01T17:43:45.921552Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_wc_min = 280.5251997335418 \ntrain_wc_max = 283.27255829113597\ntest_wc = simpleScale(white_curve,train_wc_min,train_wc_max)\n\npred_test_wc = []\nfor i in range(5):\n i = i+1\n print('model :',i)\n checkpoint_filepath = f'/kaggle/input/airel20242dcnn/keras/default/8/model_1dcnn_1_{i}.keras'\n model_wc = load_model(checkpoint_filepath)\n spectra_test_wc = model_wc.predict(test_wc).flatten()\n min_train_wc = 0.0003986606214081004 \n max_train_wc = 0.007353670557466299\n spectra_test_wc = unstandardizing(spectra_test_wc, min_train_wc, max_train_wc)\n spectra_test = np.repeat(np.array(spectra_test_wc), 283).reshape((len(spectra_test_wc), 283)) \n pred_test_wc = np.append(pred_test_wc,spectra_test)\nfor i in range(5):\n i = i+1\n print('model :',i)\n checkpoint_filepath = f'/kaggle/input/ariel-1d-cnn/keras/default/1/model_1dcnn_2_{i}.keras'\n model_wc = load_model(checkpoint_filepath)\n spectra_test_wc = model_wc.predict(test_wc).flatten()\n min_train_wc = 0.0003986606214081004 \n max_train_wc = 0.007353670557466299\n spectra_test_wc = unstandardizing(spectra_test_wc, min_train_wc, max_train_wc)\n spectra_test = np.repeat(np.array(spectra_test_wc), 283).reshape((len(spectra_test_wc), 283)) \n pred_test_wc = np.append(pred_test_wc,spectra_test)\n \npred_test_wc = pred_test_wc.reshape(10,dataset.shape[0],dataset.shape[2])\npred_test_wc_cmb = pred_test_wc.mean(axis=0)\npred_test_wc_std = pred_test_wc.std(axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:45.924041Z","iopub.execute_input":"2024-11-01T17:43:45.924392Z","iopub.status.idle":"2024-11-01T17:43:54.988881Z","shell.execute_reply.started":"2024-11-01T17:43:45.924358Z","shell.execute_reply":"2024-11-01T17:43:54.987901Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Do not use CNN results on target mean but use the diff between polynomial fit and cnn \n# to estimate sigma1d\nwc_cnn = False\nif wc_cnn:\n train_s = train_s*0.7 + pred_test_wc_cmb*0.3\n\nsigma1d = np.maximum(0.00005,np.abs(pred_test_wc_cmb-train_s))\n#print(sigma1d)","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:54.990362Z","iopub.execute_input":"2024-11-01T17:43:54.990708Z","iopub.status.idle":"2024-11-01T17:43:54.996376Z","shell.execute_reply.started":"2024-11-01T17:43:54.99068Z","shell.execute_reply":"2024-11-01T17:43:54.99518Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2D CNN for residual","metadata":{}},{"cell_type":"code","source":"# data preprocess for cnn\n\ndef norm_star_spectrum (signal) : \n img_star = signal[:,:50].mean(axis = 1) + signal[:,-50:].mean(axis = 1)\n return signal/img_star[:,np.newaxis,:]\n\ndataset_norm = norm_star_spectrum(dataset)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:54.998175Z","iopub.execute_input":"2024-11-01T17:43:54.998971Z","iopub.status.idle":"2024-11-01T17:43:55.076377Z","shell.execute_reply.started":"2024-11-01T17:43:54.998939Z","shell.execute_reply":"2024-11-01T17:43:55.075104Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##### Substracting the out transit signal #####\ndef suppress_out_transit (data, ingress, egress) : \n data_in = data[:, ingress:egress,:]\n return data_in\n\n\n###### Substract the mean #####\ndef substract_data_mean(data):\n data_mean = np.zeros(data.shape)\n for i in range(data.shape[0]):\n data_mean[i] = data[i] - data[i].mean()\n return data_mean\n\n\ndef data_norm(data_abs_max, data2):\n data2 = data2/data_abs_max\n return data2\n \n","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:55.077749Z","iopub.execute_input":"2024-11-01T17:43:55.078139Z","iopub.status.idle":"2024-11-01T17:43:55.091639Z","shell.execute_reply.started":"2024-11-01T17:43:55.078108Z","shell.execute_reply":"2024-11-01T17:43:55.090136Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# data_abs_max was obtained from train data\ndata_abs_max=0.010073510664951268\ningress, egress = 75,115\ndata_obs_in = suppress_out_transit(dataset_norm, ingress, egress)\ndata_obs_2d_mean = substract_data_mean(data_obs_in)\ndata_obs_norm = data_norm(data_abs_max, data_obs_2d_mean).clip(-1.0,1.0)","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:55.093253Z","iopub.execute_input":"2024-11-01T17:43:55.093597Z","iopub.status.idle":"2024-11-01T17:43:55.103718Z","shell.execute_reply.started":"2024-11-01T17:43:55.093568Z","shell.execute_reply":"2024-11-01T17:43:55.102583Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def targets_norm_back (data, data_abs_max) : \n return data * data_abs_max\n","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:55.10809Z","iopub.execute_input":"2024-11-01T17:43:55.108574Z","iopub.status.idle":"2024-11-01T17:43:55.116898Z","shell.execute_reply.started":"2024-11-01T17:43:55.108528Z","shell.execute_reply":"2024-11-01T17:43:55.115728Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# predict target residual using assemble of CNN models\n\n# the 2D CNN models were trained using the same model stucture as the Host's notebook, but \n# reduce learning rate to 0.0001. Each model was trained by different seed\n\n\npred_cnn = []\ntargets_abs_max_list = [0.0024692340862755954, 0.003411623763961695, 0.003411623763961695, 0.0024692340862755954,\n 0.003411623763961695]\nfor i, targets_abs_max in list(enumerate(targets_abs_max_list)):\n i = i+1\n print (\"model \",i)\n if i == 1:\n checkpoint_filepath = '/kaggle/input/airel20242dcnn/keras/default/9/model_2dcnn.keras'\n else:\n checkpoint_filepath = f'/kaggle/input/airel20242dcnn/keras/default/9/model_2dcnn_1_{i}.keras'\n model = load_model(checkpoint_filepath)\n \n pred_data_norm = model.predict([data_obs_norm])\n pred_data = targets_norm_back(pred_data_norm, targets_abs_max)\n pred_cnn=np.append(pred_cnn,pred_data)\n\ningress, egress = 75,110\ndata_obs_in = suppress_out_transit(dataset_norm, ingress, egress)\ndata_obs_2d_mean = substract_data_mean(data_obs_in)\ndata_obs_norm = data_norm(data_abs_max, data_obs_2d_mean).clip(-1.0,1.0)\n\nfor i, targets_abs_max in list(enumerate(targets_abs_max_list)):\n i = i+1\n print (\"model \",i)\n checkpoint_filepath = f'/kaggle/input/airel20242dcnn/keras/default/9/model_2dcnn_2_{i}.keras'\n model = load_model(checkpoint_filepath)\n \n pred_data_norm = model.predict([data_obs_norm])\n pred_data = targets_norm_back(pred_data_norm, targets_abs_max)\n pred_cnn=np.append(pred_cnn,pred_data)\n\n\nfor i in range(5):\n i = i+1\n print(\"model \",i)\n targets_abs_max = 0.003411623763961695\n checkpoint_filepath = f'/kaggle/input/airel20242dcnn/keras/default/10/model_2dcnn_final_{i}.keras'\n model = load_model(checkpoint_filepath)\n pred_data_norm = model.predict([data_obs_norm])\n pred_data = targets_norm_back(pred_data_norm, targets_abs_max)\n pred_cnn=np.append(pred_cnn,pred_data)\npred_cnn = pred_cnn.reshape(15,data_obs_norm.shape[0],data_obs_norm.shape[2])\npred_cnn_cmb = pred_cnn.mean(axis=0)\npred_cnn_std = pred_cnn.std(axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:43:55.118404Z","iopub.execute_input":"2024-11-01T17:43:55.11875Z","iopub.status.idle":"2024-11-01T17:44:46.719321Z","shell.execute_reply.started":"2024-11-01T17:43:55.118721Z","shell.execute_reply":"2024-11-01T17:44:46.718316Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Combination of 1D and 2d for final prediction ","metadata":{}},{"cell_type":"code","source":"# Combined mean + residual \nss = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/sample_submission.csv')\ndiffAbsPctWL = pickle.load(open(\"/kaggle/input/ariel2024-sigma-estimate/ariel_sigma.pkl\", \"rb\"))\n\nshrink = 1.0\n\nscoreCorrection = True\n\nif scoreCorrection:\n preds = train_s + pred_cnn_cmb*shrink\nelse:\n preds = train_s\npreds = preds.clip(1e-8)\n\nsigma1 = sigma1d\nsigma2 = pred_cnn_std*5.0\nsigma = np.maximum(sigma1,sigma2)\nsigma = sigma.clip(0.00005, 0.000195)\n \nsubmission = pd.DataFrame(np.concatenate([preds,sigma], axis=1), columns=ss.columns[1:])\nsubmission.index = test_adc_info.index\nsubmission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:44:46.720566Z","iopub.execute_input":"2024-11-01T17:44:46.720885Z","iopub.status.idle":"2024-11-01T17:44:46.776627Z","shell.execute_reply.started":"2024-11-01T17:44:46.720858Z","shell.execute_reply":"2024-11-01T17:44:46.77544Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Making submission","metadata":{}},{"cell_type":"code","source":"submission\n#print(np.abs(pred_cnn_cmb)) ","metadata":{"execution":{"iopub.status.busy":"2024-11-01T17:44:46.778254Z","iopub.execute_input":"2024-11-01T17:44:46.77871Z","iopub.status.idle":"2024-11-01T17:44:46.808691Z","shell.execute_reply.started":"2024-11-01T17:44:46.77867Z","shell.execute_reply":"2024-11-01T17:44:46.807665Z"},"trusted":true},"outputs":[],"execution_count":null}]}