{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.11"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":101849,"databundleVersionId":13093295,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":123.041003,"end_time":"2024-08-29T13:27:04.243126","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-08-29T13:25:01.202123","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook (approximately) reproduces the plots in [this discussion](https://www.kaggle.com/competitions/ariel-data-challenge-2025/discussion/602425). It suggests that the measurements are not consistent with the labels, by demonstrating that their is an apparent transit depth difference for two transits of the same planet. **Note that this is not an issue for multiple-transit planets specifically; I'm just using one to demonstrate the core issue that the measurements are not consistent with the labels.** Note that it's an *apparent* difference - there may be a reasonable explanation that we have yet to find.\n\nI load the data with the functions provided by the hosts, with the following differences:\n* Changed time binning from 30 to 5\n* Remove cosmic rays\n* Load 2 transits from planet 1349926825, rather than 1 transit for all planets\n\nAt the end I plot the ratio of the two transits, showing a wavelength-dependent offset only in the transit zone.","metadata":{"papermill":{"duration":1.619176,"end_time":"2024-08-29T13:25:06.963493","exception":false,"start_time":"2024-08-29T13:25:05.344317","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-08-30T08:22:27.186585Z","iopub.execute_input":"2025-08-30T08:22:27.186963Z","iopub.status.idle":"2025-08-30T08:22:30.737157Z","shell.execute_reply.started":"2025-08-30T08:22:27.186931Z","shell.execute_reply":"2025-08-30T08:22:30.735837Z"}}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport itertools\nimport os\nimport glob \nfrom astropy.stats import sigma_clip\n\nfrom tqdm import tqdm","metadata":{"papermill":{"duration":1.619176,"end_time":"2024-08-29T13:25:06.963493","exception":false,"start_time":"2024-08-29T13:25:05.344317","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:22.544479Z","iopub.execute_input":"2025-08-30T09:28:22.544782Z","iopub.status.idle":"2025-08-30T09:28:25.25077Z","shell.execute_reply.started":"2025-08-30T09:28:22.54475Z","shell.execute_reply":"2025-08-30T09:28:25.249655Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\npath_folder = '/kaggle/input/ariel-data-challenge-2025/' # path to the folder containing the data\npath_out = '/kaggle/tmp/data_light_raw/' # path to the folder to store the light data\noutput_dir = '/kaggle/tmp/data_light_raw/' # path for the output directory","metadata":{"papermill":{"duration":0.021737,"end_time":"2024-08-29T13:25:07.023919","exception":false,"start_time":"2024-08-29T13:25:07.002182","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.253077Z","iopub.execute_input":"2025-08-30T09:28:25.253946Z","iopub.status.idle":"2025-08-30T09:28:25.258822Z","shell.execute_reply.started":"2025-08-30T09:28:25.253906Z","shell.execute_reply":"2025-08-30T09:28:25.257815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not os.path.exists(path_out):\n    os.makedirs(path_out)\n    print(f\"Directory {path_out} created.\")\nelse:\n    print(f\"Directory {path_out} already exists.\")\n","metadata":{"papermill":{"duration":0.024159,"end_time":"2024-08-29T13:25:07.08616","exception":false,"start_time":"2024-08-29T13:25:07.062001","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.26048Z","iopub.execute_input":"2025-08-30T09:28:25.260877Z","iopub.status.idle":"2025-08-30T09:28:25.311937Z","shell.execute_reply.started":"2025-08-30T09:28:25.260838Z","shell.execute_reply":"2025-08-30T09:28:25.309284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CHUNKS_SIZE = 1","metadata":{"papermill":{"duration":0.021663,"end_time":"2024-08-29T13:25:07.146745","exception":false,"start_time":"2024-08-29T13:25:07.125082","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.313143Z","iopub.execute_input":"2025-08-30T09:28:25.313498Z","iopub.status.idle":"2025-08-30T09:28:25.330538Z","shell.execute_reply.started":"2025-08-30T09:28:25.313475Z","shell.execute_reply":"2025-08-30T09:28:25.329409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inpainting function used in cosmic ray correction\ndef inpaint_along_axis_inplace(arr, axis=0):   \n    ndim = arr.ndim\n    T = arr.shape[axis]\n\n    # Move the interpolation axis to the front\n    arr_moved = np.moveaxis(arr, axis, 0)  # shape (T, ...)\n    flat_arr = arr_moved.reshape(T, -1)    # shape (T, N)\n\n    nan_mask = np.isnan(flat_arr)\n    needs_interp = np.any(nan_mask, axis=0)\n\n    x = np.arange(T)\n\n    for i in np.where(needs_interp)[0]:\n        col = flat_arr[:, i]\n        valid_mask = ~np.isnan(col)\n        valid_x = x[valid_mask]\n        valid_y = col[valid_mask]\n\n        if valid_x.size == 0:\n            flat_arr[:, i] = np.nan\n        else:\n            flat_arr[:, i] = np.interp(x, valid_x, valid_y)\n\n    # No need to move axes back — arr was modified in-place through views","metadata":{"papermill":{"duration":0.023531,"end_time":"2024-08-29T13:25:07.207817","exception":false,"start_time":"2024-08-29T13:25:07.184286","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.333196Z","iopub.execute_input":"2025-08-30T09:28:25.334151Z","iopub.status.idle":"2025-08-30T09:28:25.352789Z","shell.execute_reply.started":"2025-08-30T09:28:25.334111Z","shell.execute_reply":"2025-08-30T09:28:25.351515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def ADC_convert(signal, gain=0.4369, offset=-1000):\n    \"\"\"The Analog-to-Digital Conversion (adc) is performed by the detector to convert\n    the pixel voltage into an integer number. Since we are using the same conversion number \n    this year, we have simply hard-coded it inside. \"\"\"\n    signal = signal.astype(np.float64)\n    signal /= gain\n    signal += offset\n    return signal","metadata":{"papermill":{"duration":0.023531,"end_time":"2024-08-29T13:25:07.207817","exception":false,"start_time":"2024-08-29T13:25:07.184286","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.353843Z","iopub.execute_input":"2025-08-30T09:28:25.354161Z","iopub.status.idle":"2025-08-30T09:28:25.373911Z","shell.execute_reply.started":"2025-08-30T09:28:25.354137Z","shell.execute_reply":"2025-08-30T09:28:25.37248Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mask_hot_dead(signal, dead, dark):\n    hot = sigma_clip(\n        dark, sigma=5, maxiters=5\n    ).mask\n    hot = np.tile(hot, (signal.shape[0], 1, 1))\n    dead = np.tile(dead, (signal.shape[0], 1, 1))\n    signal = np.ma.masked_where(dead, signal)\n    signal = np.ma.masked_where(hot, signal)\n    return signal","metadata":{"papermill":{"duration":0.023597,"end_time":"2024-08-29T13:25:07.268543","exception":false,"start_time":"2024-08-29T13:25:07.244946","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.375112Z","iopub.execute_input":"2025-08-30T09:28:25.375484Z","iopub.status.idle":"2025-08-30T09:28:25.39579Z","shell.execute_reply.started":"2025-08-30T09:28:25.375453Z","shell.execute_reply":"2025-08-30T09:28:25.394789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def apply_linear_corr(linear_corr,clean_signal):\n    linear_corr = np.flip(linear_corr, axis=0)\n    for x, y in itertools.product(\n                range(clean_signal.shape[1]), range(clean_signal.shape[2])\n            ):\n        poli = np.poly1d(linear_corr[:, x, y])\n        clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n    return clean_signal\n    ","metadata":{"papermill":{"duration":0.032924,"end_time":"2024-08-29T13:25:07.375474","exception":false,"start_time":"2024-08-29T13:25:07.34255","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.396688Z","iopub.execute_input":"2025-08-30T09:28:25.396989Z","iopub.status.idle":"2025-08-30T09:28:25.422341Z","shell.execute_reply.started":"2025-08-30T09:28:25.396967Z","shell.execute_reply":"2025-08-30T09:28:25.421197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_dark(signal, dead, dark, dt):\n\n    dark = np.ma.masked_where(dead, dark)\n    dark = np.tile(dark, (signal.shape[0], 1, 1))\n\n    signal -= dark* dt[:, np.newaxis, np.newaxis]\n    return signal\n","metadata":{"papermill":{"duration":0.028125,"end_time":"2024-08-29T13:25:07.441764","exception":false,"start_time":"2024-08-29T13:25:07.413639","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.423226Z","iopub.execute_input":"2025-08-30T09:28:25.423542Z","iopub.status.idle":"2025-08-30T09:28:25.444316Z","shell.execute_reply.started":"2025-08-30T09:28:25.423513Z","shell.execute_reply":"2025-08-30T09:28:25.443213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_cds(signal):\n    cds = signal[:,1::2,:,:] - signal[:,::2,:,:]\n    return cds","metadata":{"papermill":{"duration":0.022926,"end_time":"2024-08-29T13:25:07.541562","exception":false,"start_time":"2024-08-29T13:25:07.518636","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.445428Z","iopub.execute_input":"2025-08-30T09:28:25.445736Z","iopub.status.idle":"2025-08-30T09:28:25.465165Z","shell.execute_reply.started":"2025-08-30T09:28:25.445709Z","shell.execute_reply":"2025-08-30T09:28:25.464126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def bin_obs(cds_signal,binning):\n    cds_transposed = cds_signal.transpose(0,1,3,2)\n    cds_binned = np.zeros((cds_transposed.shape[0], cds_transposed.shape[1]//binning, cds_transposed.shape[2], cds_transposed.shape[3]))\n    for i in range(cds_transposed.shape[1]//binning):\n        cds_binned[:,i,:,:] = np.sum(cds_transposed[:,i*binning:(i+1)*binning,:,:], axis=1)\n    return cds_binned","metadata":{"papermill":{"duration":0.023997,"end_time":"2024-08-29T13:25:07.603229","exception":false,"start_time":"2024-08-29T13:25:07.579232","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.466363Z","iopub.execute_input":"2025-08-30T09:28:25.466713Z","iopub.status.idle":"2025-08-30T09:28:25.487881Z","shell.execute_reply.started":"2025-08-30T09:28:25.466682Z","shell.execute_reply":"2025-08-30T09:28:25.486871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def correct_flat_field(flat,dead, signal):\n    flat = flat.transpose(1, 0)\n    dead = dead.transpose(1, 0)\n    flat = np.ma.masked_where(dead, flat)\n    flat = np.tile(flat, (signal.shape[0], 1, 1))\n    signal = signal / flat\n    return signal","metadata":{"papermill":{"duration":0.025161,"end_time":"2024-08-29T13:25:07.698534","exception":false,"start_time":"2024-08-29T13:25:07.673373","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.488889Z","iopub.execute_input":"2025-08-30T09:28:25.489207Z","iopub.status.idle":"2025-08-30T09:28:25.510007Z","shell.execute_reply.started":"2025-08-30T09:28:25.489172Z","shell.execute_reply":"2025-08-30T09:28:25.509059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## we will start by getting the index of the training data:\ndef get_index(files,CHUNKS_SIZE ):\n    index = []\n    for file in files:\n        file_name = file.split('/')[-1]\n        if file_name.split('_')[0] == 'AIRS-CH0' and file_name.split('_')[1] == 'signal' and file_name.split('_')[2] == '0.parquet':\n            file_index = os.path.basename(os.path.dirname(file))\n            index.append(int(file_index))\n    index = np.array(index)\n    index = np.sort(index) \n    # credit to DennisSakva\n    index=np.array_split(index, len(index)//CHUNKS_SIZE)\n    \n    return index","metadata":{"papermill":{"duration":0.024197,"end_time":"2024-08-29T13:25:07.789483","exception":false,"start_time":"2024-08-29T13:25:07.765286","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.510914Z","iopub.execute_input":"2025-08-30T09:28:25.511209Z","iopub.status.idle":"2025-08-30T09:28:25.531423Z","shell.execute_reply.started":"2025-08-30T09:28:25.511184Z","shell.execute_reply":"2025-08-30T09:28:25.53041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"files = glob.glob(os.path.join(path_folder + 'train/', '1349926825/*'))\n\nindex = get_index(files,CHUNKS_SIZE) \n\naxis_info = pd.read_parquet(os.path.join(path_folder,'axis_info.parquet'))\nDO_MASK = True\nDO_THE_NL_CORR = False\nDO_DARK = True\nDO_FLAT = True\nTIME_BINNING = True\n\ncut_inf, cut_sup = 39, 321\nl = cut_sup - cut_inf\n\nfor n, transit_str in enumerate(tqdm(['0','1'])):\n    index_chunk = ['1349926825']\n    AIRS_CH0_clean = np.zeros((CHUNKS_SIZE, 11250, 32, l))\n    FGS1_clean = np.zeros((CHUNKS_SIZE, 135000, 32, 32))\n    \n    for i in range (CHUNKS_SIZE) : \n        df = pd.read_parquet(os.path.join(path_folder,f'train/1349926825/AIRS-CH0_signal_'+transit_str+'.parquet'))\n        signal = df.values.astype(np.float64).reshape((df.shape[0], 32, 356))\n\n        signal = ADC_convert(signal,)\n        dt_airs = axis_info['AIRS-CH0-integration_time'].dropna().values\n        dt_airs[1::2] += 0.1\n        chopped_signal = signal[:, :, cut_inf:cut_sup]\n        del signal, df\n        \n        # CLEANING THE DATA: AIRS\n        flat = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/AIRS-CH0_calibration_'+transit_str+'/flat.parquet')).values.astype(np.float64).reshape((32, 356))[:, cut_inf:cut_sup]\n        dark = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/AIRS-CH0_calibration_'+transit_str+'/dark.parquet')).values.astype(np.float64).reshape((32, 356))[:, cut_inf:cut_sup]\n        dead_airs = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/AIRS-CH0_calibration_'+transit_str+'/dead.parquet')).values.astype(np.float64).reshape((32, 356))[:, cut_inf:cut_sup]\n        linear_corr = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/AIRS-CH0_calibration_'+transit_str+'/linear_corr.parquet')).values.astype(np.float64).reshape((6, 32, 356))[:, :, cut_inf:cut_sup]\n        \n        if DO_MASK:\n            chopped_signal = mask_hot_dead(chopped_signal, dead_airs, dark)\n            AIRS_CH0_clean[i] = chopped_signal\n        else:\n            AIRS_CH0_clean[i] = chopped_signal\n            \n        if DO_THE_NL_CORR: \n            linear_corr_signal = apply_linear_corr(linear_corr,AIRS_CH0_clean[i])\n            AIRS_CH0_clean[i,:, :, :] = linear_corr_signal\n        del linear_corr\n        \n        if DO_DARK: \n            cleaned_signal = clean_dark(AIRS_CH0_clean[i], dead_airs, dark, dt_airs)\n            AIRS_CH0_clean[i] = cleaned_signal\n        else: \n            pass\n        del dark\n        \n        df = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/FGS1_signal_'+transit_str+'.parquet'))\n        fgs_signal = df.values.astype(np.float64).reshape((df.shape[0], 32, 32))\n\n        \n        fgs_signal = ADC_convert(fgs_signal, )\n        dt_fgs1 = np.ones(len(fgs_signal))*0.1\n        dt_fgs1[1::2] += 0.1\n        chopped_FGS1 = fgs_signal\n        del fgs_signal, df\n        \n        # CLEANING THE DATA: FGS1\n        flat = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/FGS1_calibration_'+transit_str+'/flat.parquet')).values.astype(np.float64).reshape((32, 32))\n        dark = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/FGS1_calibration_'+transit_str+'/dark.parquet')).values.astype(np.float64).reshape((32, 32))\n        dead_fgs1 = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/FGS1_calibration_'+transit_str+'/dead.parquet')).values.astype(np.float64).reshape((32, 32))\n        linear_corr = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/FGS1_calibration_'+transit_str+'/linear_corr.parquet')).values.astype(np.float64).reshape((6, 32, 32))\n        \n        if DO_MASK:\n            chopped_FGS1 = mask_hot_dead(chopped_FGS1, dead_fgs1, dark)\n            FGS1_clean[i] = chopped_FGS1\n        else:\n            FGS1_clean[i] = chopped_FGS1\n\n        if DO_THE_NL_CORR: \n            linear_corr_signal = apply_linear_corr(linear_corr,FGS1_clean[i])\n            FGS1_clean[i,:, :, :] = linear_corr_signal\n        del linear_corr\n        \n        if DO_DARK: \n            cleaned_signal = clean_dark(FGS1_clean[i], dead_fgs1, dark,dt_fgs1)\n            FGS1_clean[i] = cleaned_signal\n        else: \n            pass\n        del dark\n        \n    # SAVE DATA AND FREE SPACE\n    AIRS_cds = get_cds(AIRS_CH0_clean)\n    FGS1_cds = get_cds(FGS1_clean)\n    \n    del AIRS_CH0_clean, FGS1_clean\n\n    # Remove cosmic rays (AIRS only)\n    is_cosmic_ray = np.abs(AIRS_cds - np.mean(AIRS_cds,1)[:,None,:,:])/np.std(AIRS_cds,1)[:,None,:,:] > 10\n    AIRS_cds[is_cosmic_ray] = np.nan\n    inpaint_along_axis_inplace(AIRS_cds,1)\n    \n    ## (Optional) Time Binning to reduce space\n    if TIME_BINNING:\n        AIRS_cds_binned = bin_obs(AIRS_cds,binning=5)\n        FGS1_cds_binned = bin_obs(FGS1_cds,binning=5*12)\n    else:\n        AIRS_cds = AIRS_cds.transpose(0,1,3,2) ## this is important to make it consistent for flat fielding, but you can always change it\n        AIRS_cds_binned = AIRS_cds\n        FGS1_cds = FGS1_cds.transpose(0,1,3,2)\n        FGS1_cds_binned = FGS1_cds\n    \n    del AIRS_cds, FGS1_cds\n    \n    for i in range (CHUNKS_SIZE):\n        flat_airs = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/AIRS-CH0_calibration_'+transit_str+'/flat.parquet')).values.astype(np.float64).reshape((32, 356))[:, cut_inf:cut_sup]\n        flat_fgs = pd.read_parquet(os.path.join(path_folder,f'train/{index_chunk[i]}/FGS1_calibration_'+transit_str+'/flat.parquet')).values.astype(np.float64).reshape((32, 32))\n        if DO_FLAT:\n            corrected_AIRS_cds_binned = correct_flat_field(flat_airs,dead_airs, AIRS_cds_binned[i])\n            AIRS_cds_binned[i] = corrected_AIRS_cds_binned\n            corrected_FGS1_cds_binned = correct_flat_field(flat_fgs,dead_fgs1, FGS1_cds_binned[i])\n            FGS1_cds_binned[i] = corrected_FGS1_cds_binned\n        else:\n            pass\n\n    ## save data\n    np.save(os.path.join(path_out, 'AIRS_clean_train_{}.npy'.format(n)), AIRS_cds_binned)\n    np.save(os.path.join(path_out, 'FGS1_train_{}.npy'.format(n)), FGS1_cds_binned)\n    del AIRS_cds_binned\n    del FGS1_cds_binned","metadata":{"papermill":{"duration":114.333225,"end_time":"2024-08-29T13:27:02.134952","exception":false,"start_time":"2024-08-29T13:25:07.801727","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:28:25.534851Z","iopub.execute_input":"2025-08-30T09:28:25.535167Z","iopub.status.idle":"2025-08-30T09:29:04.071762Z","shell.execute_reply.started":"2025-08-30T09:28:25.535141Z","shell.execute_reply":"2025-08-30T09:29:04.070769Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Once all the chunks are saved, we concatenate them back in a single dataset. This step is simply to save HDD space, modify it as you wish. ","metadata":{"papermill":{"duration":0.014271,"end_time":"2024-08-29T13:27:02.162064","exception":false,"start_time":"2024-08-29T13:27:02.147793","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def load_data (file, chunk_size, nb_files) : \n    data0 = np.load(file + '_0.npy')\n    data_all = np.zeros((nb_files*chunk_size, data0.shape[1], data0.shape[2], data0.shape[3]))\n    data_all[:chunk_size] = data0\n    for i in range (1, nb_files) : \n        data_all[i*chunk_size:(i+1)*chunk_size] = np.load(file + '_{}.npy'.format(i))\n    return data_all \n\ndata_train = load_data(path_out + 'AIRS_clean_train', CHUNKS_SIZE, len(index)) \ndata_train_FGS = load_data(path_out + 'FGS1_train', CHUNKS_SIZE, len(index))\n","metadata":{"papermill":{"duration":0.134081,"end_time":"2024-08-29T13:27:02.309386","exception":false,"start_time":"2024-08-29T13:27:02.175305","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:29:04.072849Z","iopub.execute_input":"2025-08-30T09:29:04.073198Z","iopub.status.idle":"2025-08-30T09:29:04.200163Z","shell.execute_reply.started":"2025-08-30T09:29:04.073168Z","shell.execute_reply":"2025-08-30T09:29:04.199329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport scipy as sp\ndef post_process(data):\n    data = np.sum(data,3) # sum over AIRS rows\n    return data\nNN=25\ndat1 = post_process(np.load(path_out+ '/AIRS_clean_train_1.npy'))[0,:-NN,...] # :-NN and NN: below align the transits\ndat2 = post_process(np.load(path_out+ '/AIRS_clean_train_0.npy'))[0,NN:,...]\nratio = (dat1 / dat2)\nplt.figure(figsize=(6,6))\nplt.imshow(ratio, aspect='auto', interpolation='none')\nplt.colorbar()\nplt.figure(figsize=(6,6))\nplt.imshow(sp.ndimage.gaussian_filter(ratio,sigma=[15,4],truncate=5), aspect='auto', interpolation='none')\nplt.colorbar()","metadata":{"papermill":{"duration":0.41396,"end_time":"2024-08-29T13:27:02.981707","exception":false,"start_time":"2024-08-29T13:27:02.567747","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T09:29:04.20102Z","iopub.execute_input":"2025-08-30T09:29:04.201325Z","iopub.status.idle":"2025-08-30T09:29:05.423993Z","shell.execute_reply.started":"2025-08-30T09:29:04.201297Z","shell.execute_reply":"2025-08-30T09:29:05.422871Z"}},"outputs":[],"execution_count":null}]}