{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":9629432,"sourceType":"datasetVersion","datasetId":5846888}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip --quiet install astropy","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\nimport scipy.stats\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\nimport itertools\nimport os\nfrom astropy.stats import sigma_clip\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.metrics import r2_score, mean_squared_error\nfrom scipy.optimize import minimize","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-10-23T10:41:20.37215Z","iopub.execute_input":"2024-10-23T10:41:20.372518Z","iopub.status.idle":"2024-10-23T10:41:20.377338Z","shell.execute_reply.started":"2024-10-23T10:41:20.372489Z","shell.execute_reply":"2024-10-23T10:41:20.376494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Calibrator:\n    cut_inf = 39\n    cut_sup = 321\n    sensor_to_sizes_dict = {\n        \"AIRS-CH0\": [[11250, 32, 356], [1, 32, cut_sup - cut_inf]],\n        \"FGS1\": [[135000, 32, 32], [1, 32, 32]],\n    }\n    sensor_to_linear_corr_dict = {\"AIRS-CH0\": (6, 32, 356), \"FGS1\": (6, 32, 32)}\n\n    def __init__(self, dataset, planet_id, sensor):\n        self.dataset = dataset\n        self.planet_id = planet_id\n        self.sensor = sensor\n\n    def _apply_linear_corr(self, linear_corr, clean_signal):\n        linear_corr = np.flip(linear_corr, axis=0)\n        for x, y in itertools.product(\n            range(clean_signal.shape[1]), range(clean_signal.shape[2])\n        ):\n            poli = np.poly1d(linear_corr[:, x, y])\n            clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n        return clean_signal\n\n    def _clean_dark(self, signal, dark, dt):\n        dark = np.tile(dark, (signal.shape[0], 1, 1))\n        signal -= dark * dt[:, np.newaxis, np.newaxis]\n        return signal\n\n    def get_calibrated_signal(self):\n        signal = pd.read_parquet(\n            f\"/kaggle/input/ariel-data-challenge-2024/{self.dataset}/{self.planet_id}/{self.sensor}_signal.parquet\"\n        ).to_numpy()\n        dark_frame = pd.read_parquet(\n            f\"/kaggle/input/ariel-data-challenge-2024/{self.dataset}/{self.planet_id}/{self.sensor}_calibration/dark.parquet\",\n            engine=\"pyarrow\",\n        ).to_numpy()\n        dead_frame = pd.read_parquet(\n            f\"/kaggle/input/ariel-data-challenge-2024/{self.dataset}/{self.planet_id}/{self.sensor}_calibration/dead.parquet\",\n            engine=\"pyarrow\",\n        ).to_numpy()\n        flat_frame = pd.read_parquet(\n            f\"/kaggle/input/ariel-data-challenge-2024/{self.dataset}/{self.planet_id}/{self.sensor}_calibration/flat.parquet\",\n            engine=\"pyarrow\",\n        ).to_numpy()\n        linear_corr = (\n            pd.read_parquet(\n                f\"/kaggle/input/ariel-data-challenge-2024/{self.dataset}/{self.planet_id}/{self.sensor}_calibration/linear_corr.parquet\"\n            )\n            .values.astype(np.float64)\n            .reshape(self.sensor_to_linear_corr_dict[self.sensor])\n        )\n\n        signal = signal.reshape(self.sensor_to_sizes_dict[self.sensor][0])\n        gain = adc_info.loc[self.planet_id, f\"{self.sensor}_adc_gain\"]\n        offset = adc_info.loc[self.planet_id, f\"{self.sensor}_adc_offset\"]\n        signal = signal / gain + offset\n\n        hot = sigma_clip(dark_frame, sigma=5, maxiters=5).mask\n\n        if self.sensor == \"AIRS-CH0\":\n            signal = signal[:, :, self.cut_inf : self.cut_sup]\n            dt = np.ones(len(signal)) * 0.1\n            dt[1::2] += 4.5  # @bilzard idea\n            linear_corr = linear_corr[:, :, self.cut_inf : self.cut_sup]\n            dark_frame = dark_frame[:, self.cut_inf : self.cut_sup]\n            dead_frame = dead_frame[:, self.cut_inf : self.cut_sup]\n            flat_frame = flat_frame[:, self.cut_inf : self.cut_sup]\n            hot = hot[:, self.cut_inf : self.cut_sup]\n        elif self.sensor == \"FGS1\":\n            dt = np.ones(len(signal)) * 0.1\n            dt[1::2] += 0.1\n\n        signal = signal.clip(0)  # @graySnow idea\n        linear_corr_signal = self._apply_linear_corr(linear_corr, signal)\n        signal = self._clean_dark(linear_corr_signal, dark_frame, dt)\n\n        flat = flat_frame.reshape(self.sensor_to_sizes_dict[self.sensor][1])\n        flat[dead_frame.reshape(self.sensor_to_sizes_dict[self.sensor][1])] = np.nan\n        flat[hot.reshape(self.sensor_to_sizes_dict[self.sensor][1])] = np.nan\n        signal = signal / flat\n        return signal\n\n\nclass Preprocessor:\n    sensor_to_binning = {\"AIRS-CH0\": 30, \"FGS1\": 30 * 12}\n    sensor_to_binned_dict = {\n        \"AIRS-CH0\": [11250 // sensor_to_binning[\"AIRS-CH0\"] // 2, 282],\n        \"FGS1\": [135000 // sensor_to_binning[\"FGS1\"] // 2],\n    }\n\n    def __init__(self, dataset, planet_id, sensor):\n        self.dataset = dataset\n        self.planet_id = planet_id\n        self.sensor = sensor\n        self.binning = self.sensor_to_binning[sensor]\n\n    def preprocess_signal(self):\n        signal = Calibrator(\n            dataset=self.dataset, planet_id=self.planet_id, sensor=self.sensor\n        ).get_calibrated_signal()\n\n        if self.sensor == \"AIRS-CH0\":\n            signal = signal[:, 10:22, :]\n        elif self.sensor == \"FGS1\":\n            signal = signal[:, 10:22, 10:22]\n            signal = signal.reshape(\n                signal.shape[0], signal.shape[1] * signal.shape[2]\n            )\n\n        mean_signal = np.nanmean(signal, axis=1)\n        cds_signal = mean_signal[1::2] - mean_signal[0::2]\n\n        binned = np.zeros((self.sensor_to_binned_dict[self.sensor]))\n        for j in range(cds_signal.shape[0] // self.binning):\n            binned[j] = cds_signal[\n                j * self.binning : j * self.binning + self.binning\n            ].mean(axis=0)\n\n        if self.sensor == \"FGS1\":\n            binned = binned.reshape((binned.shape[0], 1))\n\n        return binned\n\n\ndef preprocessor(x):\n    return Preprocessor(**x).preprocess_signal()","metadata":{"execution":{"iopub.status.busy":"2024-10-23T10:41:21.462836Z","iopub.execute_input":"2024-10-23T10:41:21.463305Z","iopub.status.idle":"2024-10-23T10:41:21.482476Z","shell.execute_reply.started":"2024-10-23T10:41:21.463267Z","shell.execute_reply":"2024-10-23T10:41:21.481777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = \"train\"\nadc_info = pd.read_csv(\n    \"/kaggle/input/ariel-data-challenge-2024/\" + f\"{dataset}_adc_info.csv\",\n    index_col=\"planet_id\",\n)\naxis_info = pd.read_parquet(\"/kaggle/input/ariel-data-challenge-2024/axis_info.parquet\")\nplanet_ids = adc_info.index","metadata":{"execution":{"iopub.status.busy":"2024-10-23T10:41:21.630131Z","iopub.execute_input":"2024-10-23T10:41:21.630907Z","iopub.status.idle":"2024-10-23T10:41:21.665076Z","shell.execute_reply.started":"2024-10-23T10:41:21.630874Z","shell.execute_reply":"2024-10-23T10:41:21.664387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args_fgs1 = [\n    dict(dataset=dataset, planet_id=planet_id, sensor=\"FGS1\")\n    for planet_id in planet_ids\n]\npreprocessed_signal_fgs1 = Parallel(n_jobs=-1)(delayed(preprocessor)(item) for item in tqdm(args_fgs1))\n\nargs_airs_ch0 = [\n    dict(dataset=dataset, planet_id=planet_id, sensor=\"AIRS-CH0\")\n    for planet_id in planet_ids\n]\npreprocessed_signal_airs_ch0 = Parallel(n_jobs=-1)(delayed(preprocessor)(item) for item in tqdm(args_airs_ch0))\n\npreprocessed_signal = np.concatenate(\n    [preprocessed_signal_fgs1, preprocessed_signal_airs_ch0], axis=2\n)\nnp.save('preprocessed_signal', preprocessed_signal)\npreprocessed_signal.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-23T10:41:22.138544Z","iopub.execute_input":"2024-10-23T10:41:22.139207Z","iopub.status.idle":"2024-10-23T10:48:40.842933Z","shell.execute_reply.started":"2024-10-23T10:41:22.139175Z","shell.execute_reply":"2024-10-23T10:48:40.842097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def phase_detector(signal):\n#     MIN = np.argmin(signal[30:140]) + 30\n#     signal1 = signal[:MIN]\n#     signal2 = signal[MIN:]\n\n#     first_derivative1 = np.gradient(signal1)\n#     first_derivative1 /= first_derivative1.max()\n#     first_derivative2 = np.gradient(signal2)\n#     first_derivative2 /= first_derivative2.max()\n\n#     phase1 = np.argmin(first_derivative1)\n#     phase2 = np.argmax(first_derivative2) + MIN\n\n#     return phase1, phase2\n\n\n# def predict_spectra(signal):\n#     def objective_to_minimize(s):\n#         delta = 2\n#         power = 3\n#         x = list(range(signal.shape[0] - delta * 4))\n#         y = (\n#             signal[: phase1 - delta].tolist()\n#             + (signal[phase1 + delta : phase2 - delta] * (1 + s)).tolist()\n#             + signal[phase2 + delta :].tolist()\n#         )\n\n#         z = np.polyfit(x, y, deg=power)\n#         p = np.poly1d(z)\n#         q = np.abs(p(x) - y).mean()\n#         return q\n\n#     signal = signal[:, 1:].mean(axis=1)\n#     phase1, phase2 = phase_detector(signal)\n\n#     s = minimize(fun=objective_to_minimize, x0=[0.0001], method=\"Nelder-Mead\").x[0]\n#     return s\n\n\n# predictions_spectra = [\n#     predict_spectra(preprocessed_signal[i]) for i in range(len(preprocessed_signal))\n# ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_labels['mean'] = train_labels.mean(axis=1)\n# for cols in train_labels.columns[:-1]:\n#     train_labels[cols] = train_labels[cols] - train_labels['mean']\n# # train_labels = train_labels.drop('mean', axis=1)\n# train_labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def normalize(label, target):\n#     if target:\n#         label_max = label.max(axis=None)\n#         for cols in label.columns:\n#             label[cols] = label[cols] / label_max\n#     else:\n#         label_max = np.tile(label.max().reshape(-1, 1), (1, label.shape[1]))\n#         label = label / label_max\n#     return label, label_max","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_labels, label_max = normalize(train_labels, True)\n# train_labels, label_max","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataset, dataset_max = normalize(dataset, False)\n# dataset_max = dataset_max[0, 0]\n# dataset, dataset_max","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import mean_squared_error","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# uncertainty = mean_squared_error(np.repeat(np.array(predicted).reshape(-1, 1), 283, axis=1), train_labels.values, squared=False)\n# uncertainty","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.save('uncertainty', uncertainty)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# possible = np.repeat(np.absolute(mean), repeats=3, axis=1)\n# possible[:, 1] = possible[:, 1] - uncertainty\n# possible[:, 2] = possible[:, 2] + uncertainty","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.save('possible_mean', possible)\n# np.save(\"features\", features)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}