{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# NeurlPS-EDA\n👋 Hello, fellow Kagglers!<br>\nI hope you're all enjoying the journey of exploring exoplanet atmospheres as much as I am! 🌌 I've been working hard on my notebook, and I would love your feedback and insights. Your comments can really help improve the analysis and make it even more valuable for our community.<br>\nIf you find the notebook helpful or interesting, please consider giving it an upvote! 👍 Every bit of support counts and helps us all grow together in this amazing data science community.<br>","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## Load Packages and Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2024-08-03T02:45:17.661957Z","iopub.execute_input":"2024-08-03T02:45:17.662883Z","iopub.status.idle":"2024-08-03T02:45:17.667683Z","shell.execute_reply.started":"2024-08-03T02:45:17.662847Z","shell.execute_reply":"2024-08-03T02:45:17.666583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set up Paths","metadata":{}},{"cell_type":"code","source":"data_path = Path('/kaggle/input/ariel-data-challenge-2024')\ntrain_path = data_path / 'train'","metadata":{"execution":{"iopub.status.busy":"2024-08-03T02:45:58.428535Z","iopub.execute_input":"2024-08-03T02:45:58.428954Z","iopub.status.idle":"2024-08-03T02:45:58.433686Z","shell.execute_reply.started":"2024-08-03T02:45:58.428909Z","shell.execute_reply":"2024-08-03T02:45:58.432632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load metadata","metadata":{}},{"cell_type":"code","source":"train_adc_info = pd.read_csv(data_path / 'train_adc_info.csv')\ntrain_labels = pd.read_csv(data_path / 'train_labels.csv')\naxis_info = pd.read_parquet(data_path / 'axis_info.parquet')\nwavelength = pd.read_csv(data_path / 'wavelengths.csv')","metadata":{"execution":{"iopub.status.busy":"2024-08-03T02:46:50.702235Z","iopub.execute_input":"2024-08-03T02:46:50.702613Z","iopub.status.idle":"2024-08-03T02:46:50.79275Z","shell.execute_reply.started":"2024-08-03T02:46:50.702581Z","shell.execute_reply":"2024-08-03T02:46:50.791599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Function to load and preprocess signal data","metadata":{}},{"cell_type":"code","source":"def load_signal_data(planet_id, instrument):\n    file_path = train_path / str(planet_id) / f'{instrument}_signal.parquet'\n    df = pd.read_parquet(file_path)\n    \n    # Restore full dynamic range\n    gain = train_adc_info.loc[train_adc_info['planet_id'] == planet_id, f'{instrument}_adc_gain'].values[0]\n    offset = train_adc_info.loc[train_adc_info['planet_id'] == planet_id, f'{instrument}_adc_offset'].values[0]\n    df = df * gain + offset\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-08-03T02:53:31.752953Z","iopub.execute_input":"2024-08-03T02:53:31.753877Z","iopub.status.idle":"2024-08-03T02:53:31.759697Z","shell.execute_reply.started":"2024-08-03T02:53:31.753841Z","shell.execute_reply":"2024-08-03T02:53:31.758538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data for sample planet","metadata":{}},{"cell_type":"code","source":"sample_planet_id = train_adc_info['planet_id'].iloc[0]\nairs_ch0_data = load_signal_data(sample_planet_id, 'AIRS-CH0')\nfgs1_data = load_signal_data(sample_planet_id, 'FGS1')","metadata":{"execution":{"iopub.status.busy":"2024-08-03T02:53:33.23716Z","iopub.execute_input":"2024-08-03T02:53:33.23752Z","iopub.status.idle":"2024-08-03T02:53:35.842223Z","shell.execute_reply.started":"2024-08-03T02:53:33.237492Z","shell.execute_reply":"2024-08-03T02:53:35.841045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Basic statistics","metadata":{}},{"cell_type":"code","source":"print(\"AIRS-CH0 data shape:\", airs_ch0_data.shape)\nprint(\"FGS1 data shape:\", fgs1_data.shape)\nprint(\"\\nAIRS-CH0 statistics:\")\nprint(airs_ch0_data.describe())\nprint(\"\\nFGS1 statistics:\")\nprint(fgs1_data.describe())","metadata":{"execution":{"iopub.status.busy":"2024-08-03T02:54:06.16196Z","iopub.execute_input":"2024-08-03T02:54:06.162385Z","iopub.status.idle":"2024-08-03T02:54:37.977329Z","shell.execute_reply.started":"2024-08-03T02:54:06.162352Z","shell.execute_reply":"2024-08-03T02:54:37.976026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizations","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\n\n# AIRS-CH0 Heatmap\nplt.subplot(121)\nsns.heatmap(airs_ch0_data.iloc[0].values.reshape(32, 356), cmap='viridis')\nplt.title('AIRS-CH0 First Frame')\n\n# FGS1 heatmap\nplt.subplot(122)\nsns.heatmap(fgs1_data.iloc[0].values.reshape(32, 32), cmap='viridis')\nplt.title('FGS1 First Frame')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T02:56:49.376847Z","iopub.execute_input":"2024-08-03T02:56:49.37722Z","iopub.status.idle":"2024-08-03T02:56:50.840413Z","shell.execute_reply.started":"2024-08-03T02:56:49.377192Z","shell.execute_reply":"2024-08-03T02:56:50.839234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Time series plot","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nplt.plot(airs_ch0_data.mean(axis=1), label='AIRS-CH0')\nplt.plot(fgs1_data.mean(axis=1), label='FGS1')\nplt.title('Mean Signal Over Time')\nplt.xlabel('Frame')\nplt.ylabel('Mean Signal')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T02:57:52.702814Z","iopub.execute_input":"2024-08-03T02:57:52.703197Z","iopub.status.idle":"2024-08-03T02:57:54.365446Z","shell.execute_reply.started":"2024-08-03T02:57:52.703169Z","shell.execute_reply":"2024-08-03T02:57:54.364289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Wavelength distribution","metadata":{}},{"cell_type":"code","source":"wavelength","metadata":{"execution":{"iopub.status.busy":"2024-08-03T02:58:41.811765Z","iopub.execute_input":"2024-08-03T02:58:41.812639Z","iopub.status.idle":"2024-08-03T02:58:41.833906Z","shell.execute_reply.started":"2024-08-03T02:58:41.812602Z","shell.execute_reply":"2024-08-03T02:58:41.832839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wavelength.replace([np.inf, -np.inf], np.nan)\nplt.figure(figsize=(10, 5))\nsns.histplot(wavelength.dropna().iloc[0], bins=50, kde=True)\nplt.title('Distribution of Wavelengths')\nplt.xlabel('Wavelength')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T03:03:40.601069Z","iopub.execute_input":"2024-08-03T03:03:40.601484Z","iopub.status.idle":"2024-08-03T03:03:41.018781Z","shell.execute_reply.started":"2024-08-03T03:03:40.601451Z","shell.execute_reply":"2024-08-03T03:03:41.017532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correlation between instruments","metadata":{}},{"cell_type":"code","source":"airs_ch0_mean = airs_ch0_data.mean(axis=1).dropna()\nfgs1_mean = fgs1_data.mean(axis=1).dropna()\n\nmin_length = min(len(airs_ch0_mean), len(fgs1_mean))\nairs_ch0_mean = airs_ch0_mean[:min_length]\nfgs1_mean = fgs1_mean[:min_length]\n\n\ncorrelation = np.corrcoef(airs_ch0_mean, fgs1_mean)[0, 1]\nprint(f\"\\nCorrelation between AIRS-CH0 and FGS1 mean signals: {correlation:.4f}\")\n\n# Visualize the correlation\nplt.figure(figsize=(10, 6))\nplt.scatter(airs_ch0_mean, fgs1_mean, alpha=0.5)\nplt.title('AIRS-CH0 vs FGS1 Mean Signals')\nplt.xlabel('AIRS-CH0 Mean Signal')\nplt.ylabel('FGS1 Mean Signal')\nplt.show()\n\n# Calculate rolling correlation\nwindow_size = 100  # Adjust as needed\nrolling_corr = airs_ch0_mean.rolling(window=window_size).corr(fgs1_mean)\n\nplt.figure(figsize=(12, 6))\nplt.plot(rolling_corr)\nplt.title(f'Rolling Correlation (Window Size: {window_size})')\nplt.xlabel('Frame')\nplt.ylabel('Correlation')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T03:14:28.228749Z","iopub.execute_input":"2024-08-03T03:14:28.22916Z","iopub.status.idle":"2024-08-03T03:14:29.744993Z","shell.execute_reply.started":"2024-08-03T03:14:28.229128Z","shell.execute_reply":"2024-08-03T03:14:29.743845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examine Labels","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nsns.lineplot(data=train_labels.iloc[:, 1:].T, legend=False)\nplt.title('Ground Truth Spectra')\nplt.xlabel('Wavelength Index')\nplt.ylabel('Spectrum Value')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T03:11:05.414699Z","iopub.execute_input":"2024-08-03T03:11:05.415153Z","iopub.status.idle":"2024-08-03T03:12:18.420782Z","shell.execute_reply.started":"2024-08-03T03:11:05.415118Z","shell.execute_reply":"2024-08-03T03:12:18.419622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Print some information about the calibration files","metadata":{}},{"cell_type":"code","source":"calibration_files = ['dark', 'dead', 'flat', 'linear_corr', 'read']\nfor cal_file in calibration_files:\n    file_path = train_path / str(sample_planet_id) / 'AIRS-CH0_calibration' / f'{cal_file}.parquet'\n    df = pd.read_parquet(file_path)\n    print(f\"\\nAIRS-CH0 {cal_file} calibration file shape:\", df.shape)\n    print(f\"AIRS-CH0 {cal_file} calibration file statistics:\")\n    print(df.describe())","metadata":{"execution":{"iopub.status.busy":"2024-08-03T03:06:38.219986Z","iopub.execute_input":"2024-08-03T03:06:38.220372Z","iopub.status.idle":"2024-08-03T03:06:40.928166Z","shell.execute_reply.started":"2024-08-03T03:06:38.220344Z","shell.execute_reply":"2024-08-03T03:06:40.926922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}