{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":9331726,"sourceType":"datasetVersion","datasetId":5628160},{"sourceId":195695360,"sourceType":"kernelVersion"}],"dockerImageVersionId":30761,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is an update of https://www.kaggle.com/code/sergeifironov/ariel-only-correlation\nfrom Sergei Fironov\n\nUpdates :\n- keep 10:22 pixels from the 32 (the image are well centred)\n- Use the derivative for the determination of the beginning and end of the signal during eclipse (idea from Reza R. Choubeh)\n- 'Simplification' of the code for minimize\n- Degree of polyfit <= 4\n- Predictions of test after training Ridge regression with the modelization results (targets predictions with modelization) and the True targets. ","metadata":{}},{"cell_type":"markdown","source":"# Librairies","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom tqdm import tqdm\nimport joblib\n\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import r2_score, mean_squared_error\nimport itertools\n\nfrom scipy.optimize import minimize\nfrom scipy import optimize\n\nfrom astropy.stats import sigma_clip","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-10T12:20:46.854085Z","iopub.execute_input":"2024-09-10T12:20:46.854536Z","iopub.status.idle":"2024-09-10T12:20:49.179273Z","shell.execute_reply.started":"2024-09-10T12:20:46.854495Z","shell.execute_reply":"2024-09-10T12:20:49.178167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = 'test'\nadc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/'+f'{dataset}_adc_info.csv',index_col='planet_id')\naxis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-09-10T12:20:49.181676Z","iopub.execute_input":"2024-09-10T12:20:49.182193Z","iopub.status.idle":"2024-09-10T12:20:49.368917Z","shell.execute_reply.started":"2024-09-10T12:20:49.182153Z","shell.execute_reply":"2024-09-10T12:20:49.367715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calibration","metadata":{}},{"cell_type":"code","source":"def apply_linear_corr(linear_corr,clean_signal):\n    linear_corr = np.flip(linear_corr, axis=0)\n    for x, y in itertools.product(\n                range(clean_signal.shape[1]), range(clean_signal.shape[2])\n            ):\n        poli = np.poly1d(linear_corr[:, x, y])\n        clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n    return clean_signal\n\ndef clean_dark(signal, dark, dt):\n    dark = np.tile(dark, (signal.shape[0], 1, 1))\n    signal -= dark* dt[:, np.newaxis, np.newaxis]\n    return signal\n\ndef preproc(dataset, adc_info, sensor, binning = 15):\n    cut_inf, cut_sup = 39, 321\n    sensor_sizes_dict = {\"AIRS-CH0\":[[11250, 32, 356], [1, 32, cut_sup-cut_inf]], \"FGS1\":[[135000, 32, 32], [1, 32, 32]]}\n    binned_dict = {\"AIRS-CH0\":[11250 // binning // 2, 282], \"FGS1\":[135000 // binning // 2]}\n    linear_corr_dict = {\"AIRS-CH0\":(6, 32, 356), \"FGS1\":(6, 32, 32)}\n    planet_ids = adc_info.index\n    \n    feats = []\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        signal = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/{sensor}_signal.parquet').to_numpy()\n        dark_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dark.parquet', engine='pyarrow').to_numpy()\n        dead_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dead.parquet', engine='pyarrow').to_numpy()\n        flat_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/flat.parquet', engine='pyarrow').to_numpy()\n        linear_corr = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/linear_corr.parquet').values.astype(np.float64).reshape(linear_corr_dict[sensor])\n\n        signal = signal.reshape(sensor_sizes_dict[sensor][0]) \n        gain = adc_info[f'{sensor}_adc_gain'].values[i]\n        offset = adc_info[f'{sensor}_adc_offset'].values[i]\n        signal = signal / gain + offset\n        \n        hot = sigma_clip(\n            dark_frame, sigma=5, maxiters=5\n        ).mask\n        \n        if sensor != \"FGS1\":\n            signal = signal[:, :, cut_inf:cut_sup] \n            dt = np.ones(len(signal))*0.1 \n            dt[1::2] += 4.5 #@bilzard idea\n            linear_corr = linear_corr[:, :, cut_inf:cut_sup]\n            dark_frame = dark_frame[:, cut_inf:cut_sup]\n            dead_frame = dead_frame[:, cut_inf:cut_sup]\n            flat_frame = flat_frame[:, cut_inf:cut_sup]\n            hot = hot[:, cut_inf:cut_sup]\n        else:\n            dt = np.ones(len(signal))*0.1\n            dt[1::2] += 0.1\n            \n        signal = signal.clip(0) #@graySnow idea\n        linear_corr_signal = apply_linear_corr(linear_corr, signal)\n        signal = clean_dark(linear_corr_signal, dark_frame, dt)\n        \n        flat = flat_frame.reshape(sensor_sizes_dict[sensor][1])\n        flat[dead_frame.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n        flat[hot.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n        signal = signal / flat\n        \n        \n        if sensor == \"FGS1\":\n            signal = signal[:,10:22,10:22] # **** updates ****\n            signal = signal.reshape(sensor_sizes_dict[sensor][0][0],144) # # **** updates ****\n\n        if sensor != \"FGS1\":\n            signal = signal[:,10:22,:] # **** updates ****\n\n        mean_signal = np.nanmean(signal, axis=1) \n        cds_signal = (mean_signal[1::2] - mean_signal[0::2])\n        \n        binned = np.zeros((binned_dict[sensor]))\n        for j in range(cds_signal.shape[0] // binning):\n            binned[j] = cds_signal[j*binning:j*binning+binning].mean(axis=0) \n                   \n        if sensor == \"FGS1\":\n            binned = binned.reshape((binned.shape[0],1))\n        \n        feats.append(binned)\n        \n    return np.stack(feats)\n    \npre_train = np.concatenate([preproc(f'{dataset}', adc_info, \"FGS1\", 30*12), preproc(f'{dataset}', adc_info, \"AIRS-CH0\", 30)], axis=2)","metadata":{"execution":{"iopub.status.busy":"2024-09-10T12:20:49.37053Z","iopub.execute_input":"2024-09-10T12:20:49.370899Z","iopub.status.idle":"2024-09-10T12:21:00.787544Z","shell.execute_reply.started":"2024-09-10T12:20:49.370862Z","shell.execute_reply":"2024-09-10T12:21:00.786442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelization","metadata":{}},{"cell_type":"code","source":"def phase_detector(signal):\n    \n    MIN = np.argmin(signal[30:140])+30\n    signal1 = signal[:MIN ]\n    signal2 = signal[MIN :]\n\n    first_derivative1 = np.gradient(signal1)\n    first_derivative1 /= first_derivative1.max()\n    first_derivative2 = np.gradient(signal2)\n    first_derivative2 /= first_derivative2.max()\n\n    phase1 = np.argmin(first_derivative1)\n    phase2 = np.argmax(first_derivative2) + MIN\n\n    return phase1, phase2\n    \ndef objective(s):\n    \n    best_q = 1e10\n    for i in range(4) :\n        delta = 2\n        x = list(range(signal.shape[0]-delta*4))\n        y = signal[:p1-delta].tolist() + (signal[p1+delta:p2 - delta] * (1 + s)).tolist() + signal[p2+delta:].tolist()\n        \n        z = np.polyfit(x, y, deg=i)\n        p = np.poly1d(z)\n        q = np.abs(p(x) - y).mean()\n    \n    if q < best_q :\n        best_q = q\n    \n    return q\n\n\nall_s = []\nfor i in tqdm(range(len(adc_info))):\n    \n    signal = pre_train[i,:,1:].mean(axis=1)\n    p1,p2 = phase_detector(signal)\n \n    r = minimize(\n                objective,\n                [0.0001],\n                method= 'Nelder-Mead'\n                  )\n    s = r.x[0]\n    all_s.append(s)\n    \nall_s = np.repeat(np.array(all_s), 283).reshape((len(all_s), 283))        ","metadata":{"execution":{"iopub.status.busy":"2024-09-10T12:21:00.788836Z","iopub.execute_input":"2024-09-10T12:21:00.789205Z","iopub.status.idle":"2024-09-10T12:21:00.847237Z","shell.execute_reply.started":"2024-09-10T12:21:00.789162Z","shell.execute_reply":"2024-09-10T12:21:00.846156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions with Ridge model","metadata":{}},{"cell_type":"code","source":"# pd.DataFrame(all_s)","metadata":{"execution":{"iopub.status.busy":"2024-09-10T12:27:04.712858Z","iopub.execute_input":"2024-09-10T12:27:04.713312Z","iopub.status.idle":"2024-09-10T12:27:04.718287Z","shell.execute_reply.started":"2024-09-10T12:27:04.713273Z","shell.execute_reply":"2024-09-10T12:27:04.71717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = joblib.load(\"/kaggle/input/adc24-meta-model-ridge/model_ridge_10_22_delta2.joblib\")\n# pred = model.predict(all_s)\n# pd.DataFrame(pred)\n# import pickle\n# with open('/kaggle/input/ad24-train-inf-ridge-addfe-lb-441/model.pickle', 'rb') as f:\n#     model = pickle.load(f)\n# pred = model.predict(all_s)\n# pd.DataFrame(pred)","metadata":{"execution":{"iopub.status.busy":"2024-09-10T12:27:06.113297Z","iopub.execute_input":"2024-09-10T12:27:06.113725Z","iopub.status.idle":"2024-09-10T12:27:06.118939Z","shell.execute_reply.started":"2024-09-10T12:27:06.113685Z","shell.execute_reply":"2024-09-10T12:27:06.117438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"ss = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/sample_submission.csv')\nsigma = np.ones_like(all_s) * 0.000145 \npred = all_s.clip(0) \nsubmission = pd.DataFrame(np.concatenate([pred,sigma], axis=1), columns=ss.columns[1:])\nsubmission.index = adc_info.index\nsubmission.to_csv('submission.csv')\nsubmission\n","metadata":{"execution":{"iopub.status.busy":"2024-09-10T12:27:17.329167Z","iopub.execute_input":"2024-09-10T12:27:17.329626Z","iopub.status.idle":"2024-09-10T12:27:17.373243Z","shell.execute_reply.started":"2024-09-10T12:27:17.329587Z","shell.execute_reply":"2024-09-10T12:27:17.372174Z"},"trusted":true},"execution_count":null,"outputs":[]}]}