{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport pyarrow.parquet as pq\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-04T11:34:39.159378Z","iopub.execute_input":"2024-09-04T11:34:39.160021Z","iopub.status.idle":"2024-09-04T11:34:39.166321Z","shell.execute_reply.started":"2024-09-04T11:34:39.159942Z","shell.execute_reply":"2024-09-04T11:34:39.164981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MetaData","metadata":{}},{"cell_type":"code","source":"train_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_adc_info.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.168793Z","iopub.execute_input":"2024-09-04T11:34:39.169323Z","iopub.status.idle":"2024-09-04T11:34:39.182775Z","shell.execute_reply.started":"2024-09-04T11:34:39.169266Z","shell.execute_reply":"2024-09-04T11:34:39.181387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/sample_submission.csv')\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.185179Z","iopub.execute_input":"2024-09-04T11:34:39.18568Z","iopub.status.idle":"2024-09-04T11:34:39.235728Z","shell.execute_reply.started":"2024-09-04T11:34:39.185598Z","shell.execute_reply":"2024-09-04T11:34:39.234487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_adc_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.237296Z","iopub.execute_input":"2024-09-04T11:34:39.237737Z","iopub.status.idle":"2024-09-04T11:34:39.252316Z","shell.execute_reply.started":"2024-09-04T11:34:39.237693Z","shell.execute_reply":"2024-09-04T11:34:39.251049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.255146Z","iopub.execute_input":"2024-09-04T11:34:39.255555Z","iopub.status.idle":"2024-09-04T11:34:39.336208Z","shell.execute_reply.started":"2024-09-04T11:34:39.255513Z","shell.execute_reply":"2024-09-04T11:34:39.335085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.337718Z","iopub.execute_input":"2024-09-04T11:34:39.338161Z","iopub.status.idle":"2024-09-04T11:34:39.367598Z","shell.execute_reply.started":"2024-09-04T11:34:39.338113Z","shell.execute_reply":"2024-09-04T11:34:39.366294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.369411Z","iopub.execute_input":"2024-09-04T11:34:39.369868Z","iopub.status.idle":"2024-09-04T11:34:39.381824Z","shell.execute_reply.started":"2024-09-04T11:34:39.369821Z","shell.execute_reply":"2024-09-04T11:34:39.380556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"axis_info = pq.read_table('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet').to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.383503Z","iopub.execute_input":"2024-09-04T11:34:39.383937Z","iopub.status.idle":"2024-09-04T11:34:39.406357Z","shell.execute_reply.started":"2024-09-04T11:34:39.383879Z","shell.execute_reply":"2024-09-04T11:34:39.405234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"axis_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.4078Z","iopub.execute_input":"2024-09-04T11:34:39.408287Z","iopub.status.idle":"2024-09-04T11:34:39.421943Z","shell.execute_reply.started":"2024-09-04T11:34:39.408231Z","shell.execute_reply":"2024-09-04T11:34:39.420569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.423607Z","iopub.execute_input":"2024-09-04T11:34:39.424026Z","iopub.status.idle":"2024-09-04T11:34:39.447545Z","shell.execute_reply.started":"2024-09-04T11:34:39.423967Z","shell.execute_reply":"2024-09-04T11:34:39.446303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wavelengths.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.451556Z","iopub.execute_input":"2024-09-04T11:34:39.452392Z","iopub.status.idle":"2024-09-04T11:34:39.476219Z","shell.execute_reply.started":"2024-09-04T11:34:39.452345Z","shell.execute_reply":"2024-09-04T11:34:39.47487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def list_files(directory):\n    for root, dirs, files in os.walk(directory):\n        for file in files:\n            print(os.path.join(root, file))\n\n\nlist_files('/kaggle/input/ariel-data-challenge-2024/train')\n","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:39.477629Z","iopub.execute_input":"2024-09-04T11:34:39.47806Z","iopub.status.idle":"2024-09-04T11:34:41.996412Z","shell.execute_reply.started":"2024-09-04T11:34:39.477982Z","shell.execute_reply":"2024-09-04T11:34:41.995095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Signals","metadata":{}},{"cell_type":"code","source":"FGS1_signal_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/FGS1_signal.parquet').to_pandas()\nFGS1_signal_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:41.997751Z","iopub.execute_input":"2024-09-04T11:34:41.998145Z","iopub.status.idle":"2024-09-04T11:34:42.7536Z","shell.execute_reply.started":"2024-09-04T11:34:41.998102Z","shell.execute_reply":"2024-09-04T11:34:42.752403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AIRS_CH0_signal_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/AIRS-CH0_signal.parquet').to_pandas()\nAIRS_CH0_signal_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:42.755151Z","iopub.execute_input":"2024-09-04T11:34:42.755527Z","iopub.status.idle":"2024-09-04T11:34:44.240102Z","shell.execute_reply.started":"2024-09-04T11:34:42.755487Z","shell.execute_reply":"2024-09-04T11:34:44.238864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calibration Data","metadata":{}},{"cell_type":"code","source":"AIRS_CH0_calibration_dead_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/AIRS-CH0_calibration/dead.parquet').to_pandas()\nAIRS_CH0_calibration_dead_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.241806Z","iopub.execute_input":"2024-09-04T11:34:44.242326Z","iopub.status.idle":"2024-09-04T11:34:44.291821Z","shell.execute_reply.started":"2024-09-04T11:34:44.242269Z","shell.execute_reply":"2024-09-04T11:34:44.289975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AIRS_CH0_calibration_linearcorr_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/AIRS-CH0_calibration/linear_corr.parquet').to_pandas()\nAIRS_CH0_calibration_linearcorr_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.294676Z","iopub.execute_input":"2024-09-04T11:34:44.295106Z","iopub.status.idle":"2024-09-04T11:34:44.349362Z","shell.execute_reply.started":"2024-09-04T11:34:44.295062Z","shell.execute_reply":"2024-09-04T11:34:44.348191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AIRS_CH0_calibration_read_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/AIRS-CH0_calibration/read.parquet').to_pandas()\nAIRS_CH0_calibration_read_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.350961Z","iopub.execute_input":"2024-09-04T11:34:44.351376Z","iopub.status.idle":"2024-09-04T11:34:44.409172Z","shell.execute_reply.started":"2024-09-04T11:34:44.351334Z","shell.execute_reply":"2024-09-04T11:34:44.407972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AIRS_CH0_calibration_flat_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/AIRS-CH0_calibration/flat.parquet').to_pandas()\nAIRS_CH0_calibration_flat_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.410641Z","iopub.execute_input":"2024-09-04T11:34:44.411064Z","iopub.status.idle":"2024-09-04T11:34:44.461711Z","shell.execute_reply.started":"2024-09-04T11:34:44.410981Z","shell.execute_reply":"2024-09-04T11:34:44.460479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AIRS_CH0_calibration_dark_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/AIRS-CH0_calibration/dark.parquet').to_pandas()\nAIRS_CH0_calibration_dark_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.463045Z","iopub.execute_input":"2024-09-04T11:34:44.463412Z","iopub.status.idle":"2024-09-04T11:34:44.52335Z","shell.execute_reply.started":"2024-09-04T11:34:44.463372Z","shell.execute_reply":"2024-09-04T11:34:44.522157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FGS1_calibration_dead_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/FGS1_calibration/dead.parquet').to_pandas()\nFGS1_calibration_dead_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.52498Z","iopub.execute_input":"2024-09-04T11:34:44.525486Z","iopub.status.idle":"2024-09-04T11:34:44.556215Z","shell.execute_reply.started":"2024-09-04T11:34:44.525422Z","shell.execute_reply":"2024-09-04T11:34:44.554946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FGS1_calibration_dead_parquet_1315791861","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.557848Z","iopub.execute_input":"2024-09-04T11:34:44.558342Z","iopub.status.idle":"2024-09-04T11:34:44.613898Z","shell.execute_reply.started":"2024-09-04T11:34:44.558286Z","shell.execute_reply":"2024-09-04T11:34:44.611921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FGS1_calibration_linearcorr_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/FGS1_calibration/linear_corr.parquet').to_pandas()\nFGS1_calibration_linearcorr_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.615536Z","iopub.execute_input":"2024-09-04T11:34:44.61591Z","iopub.status.idle":"2024-09-04T11:34:44.64984Z","shell.execute_reply.started":"2024-09-04T11:34:44.61587Z","shell.execute_reply":"2024-09-04T11:34:44.648648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FGS1_calibration_read_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/FGS1_calibration/read.parquet').to_pandas()\nFGS1_calibration_read_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.651423Z","iopub.execute_input":"2024-09-04T11:34:44.651816Z","iopub.status.idle":"2024-09-04T11:34:44.686396Z","shell.execute_reply.started":"2024-09-04T11:34:44.651776Z","shell.execute_reply":"2024-09-04T11:34:44.685086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FGS1_calibration_flat_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/FGS1_calibration/flat.parquet').to_pandas()\nFGS1_calibration_flat_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.690955Z","iopub.execute_input":"2024-09-04T11:34:44.691427Z","iopub.status.idle":"2024-09-04T11:34:44.736575Z","shell.execute_reply.started":"2024-09-04T11:34:44.691385Z","shell.execute_reply":"2024-09-04T11:34:44.73546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FGS1_calibration_dark_parquet_1315791861 = pq.read_table('/kaggle/input/ariel-data-challenge-2024/train/1315791861/FGS1_calibration/dark.parquet').to_pandas()\nFGS1_calibration_dark_parquet_1315791861.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.737967Z","iopub.execute_input":"2024-09-04T11:34:44.738377Z","iopub.status.idle":"2024-09-04T11:34:44.771938Z","shell.execute_reply.started":"2024-09-04T11:34:44.738335Z","shell.execute_reply":"2024-09-04T11:34:44.770852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nplt.imshow(AIRS_CH0_signal_parquet_1315791861.iloc[0].values.reshape(32, 356), cmap='plasma')\nplt.title('AIRS-CH0 Signal Sample')\nplt.colorbar()\nplt.show()\n\n\nplt.figure(figsize=(8, 8))\nplt.imshow(FGS1_signal_parquet_1315791861.iloc[0].values.reshape(32, 32), cmap='plasma')\nplt.title('FGS1 Signal Sample')\nplt.colorbar()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:44.7733Z","iopub.execute_input":"2024-09-04T11:34:44.773646Z","iopub.status.idle":"2024-09-04T11:34:45.619786Z","shell.execute_reply.started":"2024-09-04T11:34:44.773607Z","shell.execute_reply":"2024-09-04T11:34:45.618769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dark_frame = FGS1_calibration_dark_parquet_1315791861\ndead_pixels = FGS1_calibration_dead_parquet_1315791861\nflat_field = FGS1_calibration_flat_parquet_1315791861\nlinear_corr = FGS1_calibration_linearcorr_parquet_1315791861\nread_noise = FGS1_calibration_read_parquet_1315791861","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:45.621181Z","iopub.execute_input":"2024-09-04T11:34:45.621558Z","iopub.status.idle":"2024-09-04T11:34:45.62761Z","shell.execute_reply.started":"2024-09-04T11:34:45.621515Z","shell.execute_reply":"2024-09-04T11:34:45.626179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(FGS1_signal_parquet_1315791861.shape)\nprint(dark_frame.shape)\nprint(flat_field.shape)\nprint(linear_corr.shape)\nprint(read_noise.shape)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:45.635298Z","iopub.execute_input":"2024-09-04T11:34:45.63638Z","iopub.status.idle":"2024-09-04T11:34:45.642664Z","shell.execute_reply.started":"2024-09-04T11:34:45.636335Z","shell.execute_reply":"2024-09-04T11:34:45.641552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dark_frame_array = dark_frame.values\nflat_field_array = flat_field.values\ndead_pixel_mask = dead_pixels.values\nlinear_corr_array = linear_corr.values\nread_noise_array = read_noise.values\n\nFGS1_signal_reshaped = FGS1_signal_parquet_1315791861.values.reshape(135000, 32, 32)\n\ncorrected_FGS1_signal = (FGS1_signal_reshaped - dark_frame_array) / flat_field_array\n\n\ncorrected_FGS1_signal -= read_noise_array\n\n\nlinear_corr_array_expanded = np.zeros((135000, 32, 32))\n\n\n\nfor i in range(FGS1_signal_reshaped.shape[0]):\n    linear_corr_index = i % linear_corr_array.shape[0]  # Örneğin döngüsel seçim\n    linear_corr_expanded = np.tile(linear_corr_array[linear_corr_index, :], (32, 1))  # (32,) -> (32, 32)\n    linear_corr_array_expanded[i] = linear_corr_expanded\n\n\ncorrected_FGS1_signal *= linear_corr_array_expanded\n\n\ndead_pixel_mask = dead_pixel_mask.astype(bool)\n\n\nfor i in range(corrected_FGS1_signal.shape[0]):\n    mean_value = np.mean(corrected_FGS1_signal[i, ~dead_pixel_mask])\n    corrected_FGS1_signal[i, dead_pixel_mask] = mean_value\n","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:45.644231Z","iopub.execute_input":"2024-09-04T11:34:45.644689Z","iopub.status.idle":"2024-09-04T11:34:56.517021Z","shell.execute_reply.started":"2024-09-04T11:34:45.644635Z","shell.execute_reply":"2024-09-04T11:34:56.515627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corrected_FGS1_signal = corrected_FGS1_signal.reshape(135000, 1024)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:56.518364Z","iopub.execute_input":"2024-09-04T11:34:56.518734Z","iopub.status.idle":"2024-09-04T11:34:56.524026Z","shell.execute_reply.started":"2024-09-04T11:34:56.518694Z","shell.execute_reply":"2024-09-04T11:34:56.522803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_columns = 1024","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:56.525705Z","iopub.execute_input":"2024-09-04T11:34:56.526318Z","iopub.status.idle":"2024-09-04T11:34:56.5397Z","shell.execute_reply.started":"2024-09-04T11:34:56.526262Z","shell.execute_reply":"2024-09-04T11:34:56.538496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FGS1_signal_parquet_1315791861","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:56.541129Z","iopub.execute_input":"2024-09-04T11:34:56.541498Z","iopub.status.idle":"2024-09-04T11:34:56.576355Z","shell.execute_reply.started":"2024-09-04T11:34:56.541448Z","shell.execute_reply":"2024-09-04T11:34:56.575162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corrected_FGS1_signal = pd.DataFrame(corrected_FGS1_signal, columns=[f'Column_{i}' for i in range(num_columns)])\n","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:56.578089Z","iopub.execute_input":"2024-09-04T11:34:56.579067Z","iopub.status.idle":"2024-09-04T11:34:56.585258Z","shell.execute_reply.started":"2024-09-04T11:34:56.578989Z","shell.execute_reply":"2024-09-04T11:34:56.584096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corrected_FGS1_signal.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:56.586594Z","iopub.execute_input":"2024-09-04T11:34:56.586939Z","iopub.status.idle":"2024-09-04T11:34:56.622336Z","shell.execute_reply.started":"2024-09-04T11:34:56.5869Z","shell.execute_reply":"2024-09-04T11:34:56.621078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nplt.imshow(FGS1_signal_parquet_1315791861.iloc[0].values.reshape(32, 32), cmap='plasma')\nplt.title('FGS1 Signal Sample')\nplt.colorbar()\nplt.show()\n\n\nplt.figure(figsize=(8, 8))\nplt.imshow(corrected_FGS1_signal.iloc[0].values.reshape(32, 32), cmap='plasma')\nplt.title('Corrected FGS1 Signal Sample')\nplt.colorbar()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:56.624073Z","iopub.execute_input":"2024-09-04T11:34:56.624495Z","iopub.status.idle":"2024-09-04T11:34:57.32049Z","shell.execute_reply.started":"2024-09-04T11:34:56.624454Z","shell.execute_reply":"2024-09-04T11:34:57.319306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dark_frame_airs = AIRS_CH0_calibration_dark_parquet_1315791861\ndead_pixels_airs = AIRS_CH0_calibration_dead_parquet_1315791861\nflat_field_airs = AIRS_CH0_calibration_flat_parquet_1315791861\nlinear_corr_airs = AIRS_CH0_calibration_linearcorr_parquet_1315791861\nread_noise_airs = AIRS_CH0_calibration_read_parquet_1315791861","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:57.322074Z","iopub.execute_input":"2024-09-04T11:34:57.322464Z","iopub.status.idle":"2024-09-04T11:34:57.32814Z","shell.execute_reply.started":"2024-09-04T11:34:57.32242Z","shell.execute_reply":"2024-09-04T11:34:57.32693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(AIRS_CH0_signal_parquet_1315791861.shape)\nprint(dark_frame_airs.shape)\nprint(flat_field_airs.shape)\nprint(linear_corr_airs.shape)\nprint(read_noise_airs.shape)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:57.329599Z","iopub.execute_input":"2024-09-04T11:34:57.329966Z","iopub.status.idle":"2024-09-04T11:34:57.340602Z","shell.execute_reply.started":"2024-09-04T11:34:57.329926Z","shell.execute_reply":"2024-09-04T11:34:57.33943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dark_frame_array_airs = dark_frame_airs.values\nflat_field_array_airs = flat_field_airs.values\ndead_pixel_mask_airs = dead_pixels_airs.values\nlinear_corr_array_airs = linear_corr_airs.values\nread_noise_array_airs = read_noise_airs.values\n\n\nAIRS_CH0_signal_reshaped = AIRS_CH0_signal_parquet_1315791861.values.reshape(11250, 32, 356)\n\ncorrected_AIRS_CH0_signal = (AIRS_CH0_signal_reshaped - dark_frame_array_airs) / flat_field_array_airs\ncorrected_AIRS_CH0_signal -= read_noise_array_airs\n\ndead_pixel_mask_airs = dead_pixel_mask_airs.astype(bool)\n\n\nfor i in range(corrected_AIRS_CH0_signal.shape[0]):\n    mean_value_airs = np.mean(corrected_AIRS_CH0_signal[i, ~dead_pixel_mask_airs])\n    corrected_AIRS_CH0_signal[i, dead_pixel_mask_airs] = mean_value_airs\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:34:57.342057Z","iopub.execute_input":"2024-09-04T11:34:57.342422Z","iopub.status.idle":"2024-09-04T11:35:02.072085Z","shell.execute_reply.started":"2024-09-04T11:34:57.342383Z","shell.execute_reply":"2024-09-04T11:35:02.070972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"linear_corr_array_expanded_airs = np.zeros((11250, 32, 356))\n\nfor i in range(corrected_AIRS_CH0_signal.shape[0]):\n    linear_corr_index_airs = i % linear_corr_array_airs.shape[0] \n    linear_corr_expanded_airs = np.tile(linear_corr_array_airs[linear_corr_index_airs, :], (32, 1))  # (32,) -> (32, 32)\n    linear_corr_array_expanded_airs[i] = linear_corr_expanded_airs\n\n\ncorrected_AIRS_CH0_signal = np.maximum(corrected_AIRS_CH0_signal, 0)\ncorrected_AIRS_CH0_signal *= linear_corr_array_expanded_airs\ncorrected_AIRS_CH0_signal = np.maximum(corrected_AIRS_CH0_signal, 0)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:35:02.073503Z","iopub.execute_input":"2024-09-04T11:35:02.073982Z","iopub.status.idle":"2024-09-04T11:35:05.395469Z","shell.execute_reply.started":"2024-09-04T11:35:02.073929Z","shell.execute_reply":"2024-09-04T11:35:05.394289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corrected_AIRS_CH0_signal = corrected_AIRS_CH0_signal.reshape(11250, 11392)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:35:05.39691Z","iopub.execute_input":"2024-09-04T11:35:05.397311Z","iopub.status.idle":"2024-09-04T11:35:05.403326Z","shell.execute_reply.started":"2024-09-04T11:35:05.397271Z","shell.execute_reply":"2024-09-04T11:35:05.401918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_columns_airs = 11392","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:35:05.404822Z","iopub.execute_input":"2024-09-04T11:35:05.405214Z","iopub.status.idle":"2024-09-04T11:35:05.419827Z","shell.execute_reply.started":"2024-09-04T11:35:05.405172Z","shell.execute_reply":"2024-09-04T11:35:05.41809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corrected_AIRS_CH0_signal = pd.DataFrame(corrected_AIRS_CH0_signal, columns=[f'Column_{i}' for i in range(num_columns_airs)])","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:35:05.42193Z","iopub.execute_input":"2024-09-04T11:35:05.422458Z","iopub.status.idle":"2024-09-04T11:35:05.440199Z","shell.execute_reply.started":"2024-09-04T11:35:05.422412Z","shell.execute_reply":"2024-09-04T11:35:05.438953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corrected_AIRS_CH0_signal","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:35:05.441601Z","iopub.execute_input":"2024-09-04T11:35:05.441982Z","iopub.status.idle":"2024-09-04T11:35:05.492784Z","shell.execute_reply.started":"2024-09-04T11:35:05.441941Z","shell.execute_reply":"2024-09-04T11:35:05.491669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nplt.imshow(AIRS_CH0_signal_parquet_1315791861.iloc[0].values.reshape(32, 356), cmap='plasma')\nplt.title('AIRS-CH0 Signal Sample')\nplt.colorbar()\nplt.show()\n\nplt.figure(figsize=(12, 6))\nplt.imshow(corrected_AIRS_CH0_signal.iloc[0].values.reshape(32, 356), cmap='plasma')\nplt.title('Corrected CAIRS-CH0 Signal Sample')\nplt.colorbar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T11:35:05.494525Z","iopub.execute_input":"2024-09-04T11:35:05.495344Z","iopub.status.idle":"2024-09-04T11:35:06.311647Z","shell.execute_reply.started":"2024-09-04T11:35:05.495286Z","shell.execute_reply":"2024-09-04T11:35:06.310472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Wavelenghts","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}