{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sea\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-08-03T19:42:15.11457Z","iopub.execute_input":"2024-08-03T19:42:15.114954Z","iopub.status.idle":"2024-08-03T19:42:17.577006Z","shell.execute_reply.started":"2024-08-03T19:42:15.114924Z","shell.execute_reply":"2024-08-03T19:42:17.575745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_adc = pd.read_csv(\"/kaggle/input/ariel-data-challenge-2024/train_adc_info.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-08-03T19:42:51.806467Z","iopub.execute_input":"2024-08-03T19:42:51.80776Z","iopub.status.idle":"2024-08-03T19:42:51.830614Z","shell.execute_reply.started":"2024-08-03T19:42:51.807707Z","shell.execute_reply":"2024-08-03T19:42:51.829252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# What do the images look like?","metadata":{}},{"cell_type":"code","source":"class PlanetImages:\n    TRAIN_IMGS_FOLDER = \"/kaggle/input/ariel-data-challenge-2024/train\"\n    \n    def __init__(self, img_num):\n        self.img_num = img_num\n        self.planet_corrections = train_adc.loc[train_adc.planet_id==img_num]\n        \n        self.image_folder = f\"{PlanetImages.TRAIN_IMGS_FOLDER}/{img_num}\"\n        self.CH0_path  = f\"{self.image_folder}/AIRS-CH0_signal.parquet\"\n        self.FGS1_path = f\"{self.image_folder}/FGS1_signal.parquet\"\n        self.CH0_imgs = pd.read_parquet(self.CH0_path)   # each row is an image\n        self.FGS1_imgs = pd.read_parquet(self.FGS1_path) # each row is an image\n    \n    def process_images(self, img_stack, device):\n        \"\"\"\n        Apply image corrections which are inspired by this notebook\n        https://www.kaggle.com/code/jnesbit6/sensor-noise-correction\n        \n        device: {'CH0', 'FGS1'}\n        \"\"\"\n        if device==\"CH0\":\n            dark_frame = pd.read_parquet(f\"{PlanetImages.TRAIN_IMGS_FOLDER}/{self.img_num}/AIRS-CH0_calibration/dark.parquet\").values.reshape(1,32,356)\n            read_frame = pd.read_parquet(f\"{PlanetImages.TRAIN_IMGS_FOLDER}/{self.img_num}/AIRS-CH0_calibration/read.parquet\").values.reshape(1,32,356)\n            flat_frame = pd.read_parquet(f\"{PlanetImages.TRAIN_IMGS_FOLDER}/{self.img_num}/AIRS-CH0_calibration/flat.parquet\").values.reshape(1,32,356)\n            dead_frame = pd.read_parquet(f\"{PlanetImages.TRAIN_IMGS_FOLDER}/{self.img_num}/AIRS-CH0_calibration/dead.parquet\").values.reshape(1,32,356)\n            flat_frame[dead_frame] = 1     \n            gain   = self.planet_corrections[\"AIRS-CH0_adc_gain\"].item()   # scalar\n            offset = self.planet_corrections[\"AIRS-CH0_adc_offset\"].item() # scalar\n        else:\n            dark_frame = pd.read_parquet(f\"{PlanetImages.TRAIN_IMGS_FOLDER}/{self.img_num}/FGS1_calibration/dark.parquet\").values.reshape(1,32,32)\n            read_frame = pd.read_parquet(f\"{PlanetImages.TRAIN_IMGS_FOLDER}/{self.img_num}/FGS1_calibration/read.parquet\").values.reshape(1,32,32)\n            flat_frame = pd.read_parquet(f\"{PlanetImages.TRAIN_IMGS_FOLDER}/{self.img_num}/FGS1_calibration/flat.parquet\").values.reshape(1,32,32)\n            dead_frame = pd.read_parquet(f\"{PlanetImages.TRAIN_IMGS_FOLDER}/{self.img_num}/FGS1_calibration/dead.parquet\").values.reshape(1,32,32)\n            flat_frame[dead_frame] = 1\n            gain   = self.planet_corrections[\"FGS1_adc_gain\"].item()    # scalar\n            offset = self.planet_corrections[\"FGS1_adc_offset\"].item()  # scalar\n        ### Correction process\n        img_stack = (img_stack - dark_frame - read_frame)/flat_frame \n        img_stack = img_stack*gain + offset\n        return img_stack\n    \n    def plot_images(self, device, nrow:int, ncol:int, stack_size:int=10, figsize:tuple=(16,4)):\n        \"\"\"\n        Displays a subset of images for a given device which are CORRECTED \n        using electronic characteristics and the corresponding gain and offset values. \n        \n        device: {'CH0', 'FGS1'}\n        nrow: row count for organizing all graphs.\n        ncol: column count for organizing all graphs. \n        stack_size: how many images to select from a given device\n        \"\"\"\n        if device==\"CH0\":\n            img_stack = self.CH0_imgs.iloc[:stack_size, :].values.reshape(stack_size, 32, 356)\n            \n        elif device==\"FGS1\":\n            img_stack = self.FGS1_imgs.iloc[:stack_size, :].values.reshape(stack_size, 32, 32)\n            \n        img_stack = self.process_images(img_stack, device)\n        \n        fig,axes = plt.subplots(nrow, ncol, figsize=figsize)\n        axes = axes.ravel()\n        for k in range(stack_size):\n            img_k = img_stack[k, :, :]\n            sea.heatmap(img_k, cmap=\"mako\", square=True, ax=axes[k])\n            axes[k].set_title(f\"{device}: Image Number {k+1} for planet\\n {self.img_num}\", size=8)\n            axes[k].set_xticks([])\n            axes[k].set_yticks([])\n            axes[k].set_xticklabels([])\n            axes[k].set_yticklabels([])","metadata":{"execution":{"iopub.status.busy":"2024-08-03T20:08:44.027084Z","iopub.execute_input":"2024-08-03T20:08:44.027563Z","iopub.status.idle":"2024-08-03T20:08:44.0466Z","shell.execute_reply.started":"2024-08-03T20:08:44.027491Z","shell.execute_reply":"2024-08-03T20:08:44.045281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"planet = PlanetImages(100468857)\nplanet.plot_images(\"CH0\", nrow=2, ncol=5, stack_size=10)\nplanet.plot_images(\"FGS1\", nrow=2, ncol=5, stack_size=10)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T20:08:46.508218Z","iopub.execute_input":"2024-08-03T20:08:46.508657Z","iopub.status.idle":"2024-08-03T20:08:57.023497Z","shell.execute_reply.started":"2024-08-03T20:08:46.508623Z","shell.execute_reply":"2024-08-03T20:08:57.022414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"planet = PlanetImages(1044371083)\nplanet.plot_images(\"CH0\", nrow=2, ncol=5, stack_size=10)\nplanet.plot_images(\"FGS1\", nrow=2, ncol=5, stack_size=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How about taking the difference between consecutive images?\n\n\n<h2 style=\"font-size: 20px;\"> The images below have not been corrected through processing.\n</h2>","metadata":{}},{"cell_type":"code","source":"cho_imgs = pd.read_parquet(\"/kaggle/input/ariel-data-challenge-2024/train/100468857/AIRS-CH0_signal.parquet\")\nleading_imgs = cho_imgs.iloc[1:, :].reset_index(drop=True)\nlagging_imgs = cho_imgs.iloc[:-1, :].reset_index(drop=True)\ndiff = leading_imgs - lagging_imgs\ndiff = diff.values # pandas dataframe -> numpy matrix","metadata":{"execution":{"iopub.status.busy":"2024-08-03T19:43:29.824573Z","iopub.execute_input":"2024-08-03T19:43:29.824973Z","iopub.status.idle":"2024-08-03T19:43:31.40794Z","shell.execute_reply.started":"2024-08-03T19:43:29.824942Z","shell.execute_reply":"2024-08-03T19:43:31.406786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,axes = plt.subplots(1,2, figsize=(8,16))\naxes[0].matshow(cho_imgs, cmap=\"mako\")\naxes[0].set_xticks([])\naxes[0].set_yticks([])\naxes[0].set_title(\"Original Image Matrix for \\n 100468857\")\n\naxes[1].matshow(diff, cmap=\"mako\")\naxes[1].set_xticks([])\naxes[1].set_yticks([])\naxes[1].set_title(\"Differenced Image Matrix for \\n 100468857\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T19:43:35.872749Z","iopub.execute_input":"2024-08-03T19:43:35.874093Z","iopub.status.idle":"2024-08-03T19:43:41.908077Z","shell.execute_reply.started":"2024-08-03T19:43:35.874056Z","shell.execute_reply":"2024-08-03T19:43:41.906935Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplot(411)\ng = sea.heatmap(diff[0, :].reshape(32,356), cmap=\"mako\", square=True)\ng.set_xticks([])\ng.set_yticks([])\ng.set_xticklabels([])\ng.set_yticklabels([])\ng.set_title(\"Differenced Image Number 1\")\n\nplt.subplot(412)\ng= sea.heatmap(diff[1, :].reshape(32,356), cmap=\"mako\", square=True)\ng.set_xticks([])\ng.set_yticks([])\ng.set_xticklabels([])\ng.set_yticklabels([])\ng.set_title(\"Differenced Image Number 2\")\n\nplt.subplot(413)\ng = sea.heatmap(diff[2, :].reshape(32,356), cmap=\"mako\", square=True)\ng.set_xticks([])\ng.set_yticks([])\ng.set_xticklabels([])\ng.set_yticklabels([])\ng.set_title(\"Differenced Image Number 3\")\n\nplt.subplot(414)\ng = sea.heatmap(diff[3, :].reshape(32,356), cmap=\"mako\", square=True)\ng.set_xticks([])\ng.set_yticks([])\ng.set_xticklabels([])\ng.set_yticklabels([])\ng.set_title(\"Differenced Image Number 4\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T19:44:06.005418Z","iopub.execute_input":"2024-08-03T19:44:06.005826Z","iopub.status.idle":"2024-08-03T19:44:07.296944Z","shell.execute_reply.started":"2024-08-03T19:44:06.005793Z","shell.execute_reply":"2024-08-03T19:44:07.295738Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# What about the nature of the flattened pixels' intensities as time series?\n\n<h2 style=\"font-size: 20px;\"> The series below have not been corrected through processing.\n</h2>","metadata":{}},{"cell_type":"code","source":"fig,axes = plt.subplots(2, 3, figsize=(16,8))\naxes = axes.ravel()\n\naxes[0].plot(cho_imgs.iloc[0, :].values)\naxes[0].set_title(\"Image #1 for \\n 100468857\")\naxes[0].set_xticklabels([])\naxes[0].set_xticks([])\naxes[0].set_ylabel(\"Pixel Intensity\")\n\naxes[1].plot(cho_imgs.iloc[1, :].values)\naxes[1].set_title(\"Image #2 for \\n 100468857\")\naxes[1].set_xticklabels([])\naxes[1].set_xticks([])\n\naxes[2].plot(cho_imgs.iloc[2, :].values)\naxes[2].set_title(\"Image #3 for \\n 100468857\")\naxes[2].set_xticklabels([])\naxes[2].set_xticks([])\n\naxes[3].plot(cho_imgs.iloc[3, :].values)\naxes[3].set_title(\"Image #4 for \\n 100468857\")\naxes[3].set_ylabel(\"Pixel Intensity\")\n\naxes[4].plot(cho_imgs.iloc[4, :].values)\naxes[4].set_title(\"Image #5 for \\n 100468857\")\naxes[4].set_xlabel(\"Pixel Index\")\n\naxes[5].plot(cho_imgs.iloc[5, :].values)\naxes[5].set_title(\"Image #6 for \\n 100468857\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T20:11:30.928871Z","iopub.execute_input":"2024-08-03T20:11:30.929356Z","iopub.status.idle":"2024-08-03T20:11:32.057795Z","shell.execute_reply.started":"2024-08-03T20:11:30.929313Z","shell.execute_reply":"2024-08-03T20:11:32.056584Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,axes = plt.subplots(2, 3, figsize=(16,8))\naxes = axes.ravel()\n\naxes[0].plot(diff[0, :])\naxes[0].set_title(\"Image #2 - Image #1\")\naxes[0].set_xticklabels([])\naxes[0].set_xticks([])\naxes[0].set_ylabel(\"Pixel Intensity\")\n\naxes[1].plot(diff[1, :])\naxes[1].set_title(\"Image #3 - Image #2\")\naxes[1].set_xticklabels([])\naxes[1].set_xticks([])\n\naxes[2].plot(diff[2, :])\naxes[2].set_title(\"Image #4 - Image #3\")\naxes[2].set_xticklabels([])\naxes[2].set_xticks([])\n\naxes[3].plot(diff[3, :])\naxes[3].set_title(\"Image #5 - Image #4\")\naxes[3].set_ylabel(\"Pixel Intensity\")\n\naxes[4].plot(diff[4, :])\naxes[4].set_title(\"Image #6 - Image #5\")\naxes[4].set_xlabel(\"Pixel Index\")\n\naxes[5].plot(diff[5, :])\naxes[5].set_title(\"Image #7 - Image #6\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T19:59:41.474539Z","iopub.execute_input":"2024-08-03T19:59:41.474955Z","iopub.status.idle":"2024-08-03T19:59:42.991677Z","shell.execute_reply.started":"2024-08-03T19:59:41.474923Z","shell.execute_reply":"2024-08-03T19:59:42.99056Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"axis_info = pd.read_parquet(\"/kaggle/input/ariel-data-challenge-2024/axis_info.parquet\")\naxis_info.head(6)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T20:14:02.602719Z","iopub.execute_input":"2024-08-03T20:14:02.603419Z","iopub.status.idle":"2024-08-03T20:14:02.629134Z","shell.execute_reply.started":"2024-08-03T20:14:02.603379Z","shell.execute_reply":"2024-08-03T20:14:02.628035Z"},"trusted":true},"execution_count":null,"outputs":[]}]}