{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":101849,"databundleVersionId":13093295,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport itertools\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom astropy.stats import sigma_clip","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:08.88961Z","iopub.execute_input":"2025-09-16T16:04:08.889926Z","iopub.status.idle":"2025-09-16T16:04:13.698962Z","shell.execute_reply.started":"2025-09-16T16:04:08.889893Z","shell.execute_reply":"2025-09-16T16:04:13.698067Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Functions\n\ndef plot_signal(signal):\n\n    if signal.ndim == 3:\n        # visualize the first timestep\n        sample0 = signal[0]  # shape (32, 282)\n        sample1 = signal[1]  # shape (32, 282)\n    else:\n        # Visulaize the whole image\n        sample0 = signal  # shape (32, 282)\n        sample1 = signal  # shape (32, 282)\n    \n    fig, axes = plt.subplots(1, 2, figsize=(12, 5))\n    \n    # Plot index 0\n    im0 = axes[0].imshow(sample0, aspect='auto', cmap='viridis')\n    axes[0].set_title(\"Index 0\")\n    plt.colorbar(im0, ax=axes[0], shrink=0.7)\n    \n    # Plot index 1\n    im1 = axes[1].imshow(sample1, aspect='auto', cmap='viridis')\n    axes[1].set_title(\"Index 1\")\n    plt.colorbar(im1, ax=axes[1], shrink=0.7)\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:13.70031Z","iopub.execute_input":"2025-09-16T16:04:13.700894Z","iopub.status.idle":"2025-09-16T16:04:13.707437Z","shell.execute_reply.started":"2025-09-16T16:04:13.700868Z","shell.execute_reply":"2025-09-16T16:04:13.706327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"input_path = r'/kaggle/input/ariel-data-challenge-2025/'\n\nimage_id = 1010375142\n\ndf = pd.read_parquet(input_path + f'/train/{image_id}/AIRS-CH0_signal_0.parquet')\n\naxis_info = pd.read_parquet(input_path + 'axis_info.parquet')\nadc_info = pd.read_csv(input_path + 'adc_info.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:13.708169Z","iopub.execute_input":"2025-09-16T16:04:13.708444Z","iopub.status.idle":"2025-09-16T16:04:16.480961Z","shell.execute_reply.started":"2025-09-16T16:04:13.70841Z","shell.execute_reply":"2025-09-16T16:04:16.480115Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# What does the AIRS raw data look like?\n\n**Basic Details**\n\n- The raw data shape of AIRS signal for a planet is 11250x11392.\n- These needs to be reshaped to 11250x32x356\n- Values are in uint16 format\n\n**Time Step details**\n- These are spectrograms captured at constant time steps.\n- Spectrograms comes in pairs - correlated Data sampling (CDS).\n- The details of the timesteps are provided in axis info file.\n- Each pair are 0.000028 units* apart\n- Each subsequent pair is captured at 0.001306 units* apart\n\n(*) -> Unit is not yet clear.\n\n**Doubts**\n\n- Axis Info file mentions two separate axis for AIRS CHO signal - AIRS-CH0-axis0-h & AIRS-CH0-axis2-um\n","metadata":{}},{"cell_type":"code","source":"axis_info.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:20.703267Z","iopub.execute_input":"2025-09-16T16:04:20.703596Z","iopub.status.idle":"2025-09-16T16:04:20.729814Z","shell.execute_reply.started":"2025-09-16T16:04:20.703572Z","shell.execute_reply":"2025-09-16T16:04:20.729042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:21.998782Z","iopub.execute_input":"2025-09-16T16:04:21.999082Z","iopub.status.idle":"2025-09-16T16:04:22.028595Z","shell.execute_reply.started":"2025-09-16T16:04:21.999057Z","shell.execute_reply":"2025-09-16T16:04:22.027621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"axis_info[axis_info['AIRS-CH0-axis0-h'].notnull()]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:23.548797Z","iopub.execute_input":"2025-09-16T16:04:23.549108Z","iopub.status.idle":"2025-09-16T16:04:23.565109Z","shell.execute_reply.started":"2025-09-16T16:04:23.549079Z","shell.execute_reply":"2025-09-16T16:04:23.564348Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"diff_axis0 = axis_info['AIRS-CH0-axis0-h'].diff()\ndiff_axis0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:24.727293Z","iopub.execute_input":"2025-09-16T16:04:24.727624Z","iopub.status.idle":"2025-09-16T16:04:24.738127Z","shell.execute_reply.started":"2025-09-16T16:04:24.727599Z","shell.execute_reply":"2025-09-16T16:04:24.737262Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Processing Steps","metadata":{}},{"cell_type":"markdown","source":"## ADC Conversion\n\n**Step 1 : ADS Conversion**\n\n- This is done to refactor the signals from uint16 to float16. We can use the gain and offset information provided in the metadata.\n\n**Step 2 : Signal Chopping**\n\n- we chop the signal to the relevant wavelengths for each spectrogram image. (we chop from index 39 to 321)\n\n**Need Clarity**\n\n-  What is axis unit of the images?\n-  How did we arrive at the indices to be choosen?  ","metadata":{}},{"cell_type":"code","source":"cut_inf, cut_sup = 39, 321\nl = cut_sup - cut_inf\n\nsignal = df.values.reshape((df.shape[0], 32, 356))\nprint(signal.shape)\n\nchopped_signal = signal[:, :, cut_inf:cut_sup]\nprint(chopped_signal.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:28.436141Z","iopub.execute_input":"2025-09-16T16:04:28.436462Z","iopub.status.idle":"2025-09-16T16:04:28.442745Z","shell.execute_reply.started":"2025-09-16T16:04:28.436434Z","shell.execute_reply":"2025-09-16T16:04:28.44181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_signal(signal=chopped_signal)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:29.573158Z","iopub.execute_input":"2025-09-16T16:04:29.573451Z","iopub.status.idle":"2025-09-16T16:04:30.348548Z","shell.execute_reply.started":"2025-09-16T16:04:29.573426Z","shell.execute_reply":"2025-09-16T16:04:30.347599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define a function to perform analog to dgital conversion.\n\ndef ADC_convertion(signal, gain, offset):\n    '''\n    Function to convert a signal from Analog to Digital\n    '''\n    signal = signal / gain\n    signal = signal + offset\n\n    return signal\n\ngain = adc_info['AIRS-CH0_adc_gain'].loc[0]\noffset = adc_info['AIRS-CH0_adc_offset'].loc[0]\n\ncleaned_signal = ADC_convertion(chopped_signal, gain, offset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:30.713947Z","iopub.execute_input":"2025-09-16T16:04:30.71423Z","iopub.status.idle":"2025-09-16T16:04:31.329948Z","shell.execute_reply.started":"2025-09-16T16:04:30.714207Z","shell.execute_reply":"2025-09-16T16:04:31.328986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_signal(signal=cleaned_signal)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:32.020142Z","iopub.execute_input":"2025-09-16T16:04:32.02046Z","iopub.status.idle":"2025-09-16T16:04:32.625977Z","shell.execute_reply.started":"2025-09-16T16:04:32.020432Z","shell.execute_reply":"2025-09-16T16:04:32.625048Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Findings**\n\n- After ADC there are negative values introduced in the data.","metadata":{}},{"cell_type":"markdown","source":"## Masking hot and dead pixels.\n\n**Step 1**\n\n- We load up the dead and dark signals from the metadata available.\n- reshape to the required shapes (32,356) and chop to the required wavelength range (39, 321)","metadata":{}},{"cell_type":"code","source":"dead_airs = pd.read_parquet(input_path + f'train/{image_id}/AIRS-CH0_calibration_0/dead.parquet').values.astype(np.float64).reshape((32, 356))[:, cut_inf:cut_sup]\n\ndark = pd.read_parquet(input_path + f'train/{image_id}/AIRS-CH0_calibration_0/dark.parquet').values.astype(np.float64).reshape((32, 356))[:, cut_inf:cut_sup]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:50.167426Z","iopub.execute_input":"2025-09-16T16:04:50.16782Z","iopub.status.idle":"2025-09-16T16:04:50.236256Z","shell.execute_reply.started":"2025-09-16T16:04:50.167791Z","shell.execute_reply":"2025-09-16T16:04:50.235254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Visulaize the dead_airs and dark pixels\n\nplot_signal(signal=dead_airs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:51.509905Z","iopub.execute_input":"2025-09-16T16:04:51.510204Z","iopub.status.idle":"2025-09-16T16:04:52.064176Z","shell.execute_reply.started":"2025-09-16T16:04:51.510178Z","shell.execute_reply":"2025-09-16T16:04:52.063195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_signal(signal=dark)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:52.426956Z","iopub.execute_input":"2025-09-16T16:04:52.427236Z","iopub.status.idle":"2025-09-16T16:04:53.110472Z","shell.execute_reply.started":"2025-09-16T16:04:52.427215Z","shell.execute_reply":"2025-09-16T16:04:53.109678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(dead_airs.shape)\nprint(dark.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:53.426877Z","iopub.execute_input":"2025-09-16T16:04:53.427677Z","iopub.status.idle":"2025-09-16T16:04:53.43287Z","shell.execute_reply.started":"2025-09-16T16:04:53.427643Z","shell.execute_reply":"2025-09-16T16:04:53.431819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"AIRS_CH0_clean = np.ma.zeros((1, 11250, 32, l))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:54.087496Z","iopub.execute_input":"2025-09-16T16:04:54.087847Z","iopub.status.idle":"2025-09-16T16:04:54.092334Z","shell.execute_reply.started":"2025-09-16T16:04:54.087821Z","shell.execute_reply":"2025-09-16T16:04:54.091576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Masking of hot/dead pixels\n\ndef mask_hot_dead_signal(signal, dead, dark):\n\n    # determine the hot and dead pixels.\n    hot = sigma_clip(dark, sigma=5, maxiters=5).mask\n    \n    # Expand the hot pixel mask to match the number of frames in signal\n    hot = np.tile(hot, (signal.shape[0], 1, 1))  \n    \n    # Expand dead pixel mask too\n    dead = np.tile(dead, (signal.shape[0], 1, 1))  \n    \n    # Apply dead pixel mask\n    signal = np.ma.masked_where(dead, signal)\n    \n    # Apply hot pixel mask\n    signal = np.ma.masked_where(hot, signal)\n    \n    return signal\n\ncleaned_signal = mask_hot_dead_signal(cleaned_signal, dead_airs, dark)\n\nAIRS_CH0_clean[0] = cleaned_signal","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:54.495901Z","iopub.execute_input":"2025-09-16T16:04:54.496169Z","iopub.status.idle":"2025-09-16T16:04:58.212961Z","shell.execute_reply.started":"2025-09-16T16:04:54.496148Z","shell.execute_reply":"2025-09-16T16:04:58.212165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_signal(signal=AIRS_CH0_clean[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:58.214487Z","iopub.execute_input":"2025-09-16T16:04:58.21488Z","iopub.status.idle":"2025-09-16T16:04:58.829599Z","shell.execute_reply.started":"2025-09-16T16:04:58.214842Z","shell.execute_reply":"2025-09-16T16:04:58.828543Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Doubts**\n\n- How were the dead signal map identified in the first place?\n- Is there a better approach than standard deviation? What is th logic of the function?\n- The scale has changed after the masking - does this step take care of the negative values?","metadata":{}},{"cell_type":"markdown","source":"## Linear Correction of the signal\n\n**What is linear correction of the signal?**\n\n**Steps**\n\n- Load up the linear correction metadata information.\n- The linear correction coefficient shapes are (6, 32, 356) - chop to the correct shape.\n- for each pixel apply the linear correction - degree 5","metadata":{}},{"cell_type":"code","source":"linear_corr_raw = pd.read_parquet(input_path + f'train/{image_id}/AIRS-CH0_calibration_0/linear_corr.parquet').values.astype(np.float64).reshape((6, 32, 356))[:, :, cut_inf:cut_sup]\nlinear_corr_raw.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:04:58.830533Z","iopub.execute_input":"2025-09-16T16:04:58.830888Z","iopub.status.idle":"2025-09-16T16:04:58.878077Z","shell.execute_reply.started":"2025-09-16T16:04:58.830855Z","shell.execute_reply":"2025-09-16T16:04:58.87733Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"AIRS_CH0_clean[0].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:14:51.139247Z","iopub.execute_input":"2025-09-16T16:14:51.140097Z","iopub.status.idle":"2025-09-16T16:14:51.145374Z","shell.execute_reply.started":"2025-09-16T16:14:51.140065Z","shell.execute_reply":"2025-09-16T16:14:51.144686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying linearity correction\n\ndef apply_linear_corr(linear_corr,clean_signal):\n    linear_corr = np.flip(linear_corr, axis=0)\n    for x, y in itertools.product(\n                range(clean_signal.shape[1]), range(clean_signal.shape[2])\n            ):\n        poli = np.poly1d(linear_corr[:, x, y])\n        clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n    return clean_signal","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:20:00.977231Z","iopub.execute_input":"2025-09-16T16:20:00.977563Z","iopub.status.idle":"2025-09-16T16:20:00.983564Z","shell.execute_reply.started":"2025-09-16T16:20:00.97753Z","shell.execute_reply":"2025-09-16T16:20:00.982527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"linear_corr_signal = apply_linear_corr(linear_corr=linear_corr_raw,\n                                   clean_signal=AIRS_CH0_clean[0])\n\nAIRS_CH0_clean[0] = linear_corr_signal","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:20:09.596054Z","iopub.execute_input":"2025-09-16T16:20:09.596324Z","iopub.status.idle":"2025-09-16T16:20:39.178389Z","shell.execute_reply.started":"2025-09-16T16:20:09.596303Z","shell.execute_reply":"2025-09-16T16:20:39.177404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_signal(AIRS_CH0_clean[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:20:39.179845Z","iopub.execute_input":"2025-09-16T16:20:39.180123Z","iopub.status.idle":"2025-09-16T16:20:39.810777Z","shell.execute_reply.started":"2025-09-16T16:20:39.180099Z","shell.execute_reply":"2025-09-16T16:20:39.809694Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dark Current Correction\n\n","metadata":{}},{"cell_type":"code","source":"dark = pd.read_parquet(input_path + f'train/{image_id}/AIRS-CH0_calibration_0/dark.parquet').values.astype(np.float64).reshape((32, 356))[:, cut_inf:cut_sup]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:21:19.484661Z","iopub.execute_input":"2025-09-16T16:21:19.484945Z","iopub.status.idle":"2025-09-16T16:21:19.511596Z","shell.execute_reply.started":"2025-09-16T16:21:19.484908Z","shell.execute_reply":"2025-09-16T16:21:19.51031Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dark.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:23:32.087142Z","iopub.execute_input":"2025-09-16T16:23:32.088041Z","iopub.status.idle":"2025-09-16T16:23:32.093543Z","shell.execute_reply.started":"2025-09-16T16:23:32.088009Z","shell.execute_reply":"2025-09-16T16:23:32.092677Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_airs = axis_info['AIRS-CH0-integration_time'].dropna().values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:24:25.532778Z","iopub.execute_input":"2025-09-16T16:24:25.533096Z","iopub.status.idle":"2025-09-16T16:24:25.539004Z","shell.execute_reply.started":"2025-09-16T16:24:25.533067Z","shell.execute_reply":"2025-09-16T16:24:25.538227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_airs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:24:33.762184Z","iopub.execute_input":"2025-09-16T16:24:33.76248Z","iopub.status.idle":"2025-09-16T16:24:33.768251Z","shell.execute_reply.started":"2025-09-16T16:24:33.762453Z","shell.execute_reply":"2025-09-16T16:24:33.767554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_dark(signal, dead, dark, dt):\n\n    dark = np.ma.masked_where(dead, dark)\n    dark = np.tile(dark, (signal.shape[0], 1, 1))\n\n    signal -= dark* dt[:, np.newaxis, np.newaxis]\n    return signal","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:24:48.073073Z","iopub.execute_input":"2025-09-16T16:24:48.073378Z","iopub.status.idle":"2025-09-16T16:24:48.078329Z","shell.execute_reply.started":"2025-09-16T16:24:48.073354Z","shell.execute_reply":"2025-09-16T16:24:48.077533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"AIRS_CH0_clean[0] = clean_dark(AIRS_CH0_clean[0],dead_airs, dark, dt_airs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:25:14.200282Z","iopub.execute_input":"2025-09-16T16:25:14.200594Z","iopub.status.idle":"2025-09-16T16:25:15.369379Z","shell.execute_reply.started":"2025-09-16T16:25:14.200571Z","shell.execute_reply":"2025-09-16T16:25:15.368618Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_signal(AIRS_CH0_clean[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:25:29.995366Z","iopub.execute_input":"2025-09-16T16:25:29.99573Z","iopub.status.idle":"2025-09-16T16:25:30.607457Z","shell.execute_reply.started":"2025-09-16T16:25:29.995703Z","shell.execute_reply":"2025-09-16T16:25:30.606562Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CDS","metadata":{}},{"cell_type":"code","source":"def get_cds(signal):\n    cds = signal[:,1::2,:,:] - signal[:,::2,:,:]\n    return cds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:26:14.315226Z","iopub.execute_input":"2025-09-16T16:26:14.316147Z","iopub.status.idle":"2025-09-16T16:26:14.320723Z","shell.execute_reply.started":"2025-09-16T16:26:14.316115Z","shell.execute_reply":"2025-09-16T16:26:14.319799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"AIRS_CH0_clean[0].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:26:19.972295Z","iopub.execute_input":"2025-09-16T16:26:19.972652Z","iopub.status.idle":"2025-09-16T16:26:19.978432Z","shell.execute_reply.started":"2025-09-16T16:26:19.972623Z","shell.execute_reply":"2025-09-16T16:26:19.977417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"AIRS_cds = get_cds(AIRS_CH0_clean)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:26:54.990231Z","iopub.execute_input":"2025-09-16T16:26:54.990537Z","iopub.status.idle":"2025-09-16T16:26:55.30979Z","shell.execute_reply.started":"2025-09-16T16:26:54.990494Z","shell.execute_reply":"2025-09-16T16:26:55.308569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"AIRS_cds.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:27:25.056093Z","iopub.execute_input":"2025-09-16T16:27:25.056366Z","iopub.status.idle":"2025-09-16T16:27:25.062342Z","shell.execute_reply.started":"2025-09-16T16:27:25.056344Z","shell.execute_reply":"2025-09-16T16:27:25.061361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_signal(AIRS_cds[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:28:05.24449Z","iopub.execute_input":"2025-09-16T16:28:05.244841Z","iopub.status.idle":"2025-09-16T16:28:05.867561Z","shell.execute_reply.started":"2025-09-16T16:28:05.244817Z","shell.execute_reply":"2025-09-16T16:28:05.866553Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Time Binning\n\n","metadata":{}},{"cell_type":"code","source":"AIRS_cds.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:36:19.435027Z","iopub.execute_input":"2025-09-16T16:36:19.43533Z","iopub.status.idle":"2025-09-16T16:36:19.440916Z","shell.execute_reply.started":"2025-09-16T16:36:19.435304Z","shell.execute_reply":"2025-09-16T16:36:19.44015Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Flat Field Correction\n\n**Steps**\n- Load the flat dield data.\n- Load the dead pixel data.\n- Mask the dead pixel from the flat field data.\n- Divide each snap by the masked flat array.\n\nDoubts\n- Why transpose the shape? ","metadata":{}},{"cell_type":"code","source":"flat_airs = pd.read_parquet(input_path + f'train/{image_id}/AIRS-CH0_calibration_0/flat.parquet').values.astype(np.float64).reshape((32, 356))[:, cut_inf:cut_sup]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T17:07:10.682127Z","iopub.execute_input":"2025-09-16T17:07:10.682738Z","iopub.status.idle":"2025-09-16T17:07:10.724615Z","shell.execute_reply.started":"2025-09-16T17:07:10.682687Z","shell.execute_reply":"2025-09-16T17:07:10.723818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"flat_airs.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:38:24.372895Z","iopub.execute_input":"2025-09-16T16:38:24.373345Z","iopub.status.idle":"2025-09-16T16:38:24.381371Z","shell.execute_reply.started":"2025-09-16T16:38:24.373319Z","shell.execute_reply":"2025-09-16T16:38:24.380207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dead_airs = pd.read_parquet(input_path + f'train/{image_id}/AIRS-CH0_calibration_0/dead.parquet').values.astype(np.float64).reshape((32, 356))[:, cut_inf:cut_sup]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:37:57.655887Z","iopub.execute_input":"2025-09-16T16:37:57.65618Z","iopub.status.idle":"2025-09-16T16:37:57.694218Z","shell.execute_reply.started":"2025-09-16T16:37:57.656159Z","shell.execute_reply":"2025-09-16T16:37:57.69338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dead_airs.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:38:11.676301Z","iopub.execute_input":"2025-09-16T16:38:11.676684Z","iopub.status.idle":"2025-09-16T16:38:11.683835Z","shell.execute_reply.started":"2025-09-16T16:38:11.676649Z","shell.execute_reply":"2025-09-16T16:38:11.682796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def correct_flat_field(flat,dead, signal):\n    flat = flat.transpose(1, 0)\n    dead = dead.transpose(1, 0)\n    flat = np.ma.masked_where(dead, flat)\n    flat = np.tile(flat, (signal.shape[0], 1, 1))\n    signal = signal / flat\n    return signal","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T16:30:15.049222Z","iopub.execute_input":"2025-09-16T16:30:15.049531Z","iopub.status.idle":"2025-09-16T16:30:15.055273Z","shell.execute_reply.started":"2025-09-16T16:30:15.049488Z","shell.execute_reply":"2025-09-16T16:30:15.054115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"AIRS_cds = AIRS_cds.transpose(0,1,3,2)\nAIRS_cds_binned = AIRS_cds\n\ncorrected_AIRS_cds_binned = correct_flat_field(flat_airs,dead_airs, AIRS_cds_binned[0])\nAIRS_cds_binned[0] = corrected_AIRS_cds_binned","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T17:08:45.054418Z","iopub.execute_input":"2025-09-16T17:08:45.05485Z","iopub.status.idle":"2025-09-16T17:08:46.572863Z","shell.execute_reply.started":"2025-09-16T17:08:45.054822Z","shell.execute_reply":"2025-09-16T17:08:46.571857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_signal(AIRS_cds_binned[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T17:09:05.713676Z","iopub.execute_input":"2025-09-16T17:09:05.714384Z","iopub.status.idle":"2025-09-16T17:09:06.288645Z","shell.execute_reply.started":"2025-09-16T17:09:05.714355Z","shell.execute_reply":"2025-09-16T17:09:06.287644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(len(AIRS_cds_binned)) : \n    light_curve = AIRS_cds_binned[i,:,:,:].sum(axis=(1,2))\n    plt.plot(light_curve/light_curve.mean(), '-', alpha=0.3)\n\nplt.xlabel('Time (frame index)')\nplt.ylabel('Normalized flux in the frame')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T17:09:45.873846Z","iopub.execute_input":"2025-09-16T17:09:45.874177Z","iopub.status.idle":"2025-09-16T17:09:46.247056Z","shell.execute_reply.started":"2025-09-16T17:09:45.874151Z","shell.execute_reply":"2025-09-16T17:09:46.246174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}