{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":217766477,"sourceType":"kernelVersion"}],"dockerImageVersionId":30839,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom scipy.optimize import curve_fit\n\nimport glob\nimport os\nimport time\n# import \npath_folder = '/kaggle/input/ariel-data-challenge-2024/'\npath_out = \"/kaggle/working/\"\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T20:38:55.392987Z","iopub.execute_input":"2025-01-15T20:38:55.393288Z","iopub.status.idle":"2025-01-15T20:38:56.962814Z","shell.execute_reply.started":"2025-01-15T20:38:55.393261Z","shell.execute_reply":"2025-01-15T20:38:56.961655Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_file_paths(directory):\n    file_paths = []\n    for root, dirs, files in os.walk(directory):\n        for file in files:\n            absolute_path = os.path.abspath(os.path.join(root, file))\n            if 'AIRS_clean_train_' in absolute_path:\n                file_paths.append(absolute_path.split('AIRS_clean_train_')[1].split('.npy')[0])\n    return file_paths","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T20:38:56.963984Z","iopub.execute_input":"2025-01-15T20:38:56.964465Z","iopub.status.idle":"2025-01-15T20:38:56.970932Z","shell.execute_reply.started":"2025-01-15T20:38:56.964433Z","shell.execute_reply":"2025-01-15T20:38:56.969873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ids = get_file_paths('/kaggle/input/long-run-neurips')\n\n# for n, id in enumerate(ids):\n#     data = np.load(f'/kaggle/input/long-run-neurips/data_light_raw/AIRS_clean_train_{id}.npy')[0]\n#     data[np.isnan(data)] = 0\n    \n#     series = data.sum(axis=1).sum(axis=1)\n#     series /= np.mean(series)\n#     x_axis = np.arange(len(series))\n    \n    \n#     def taylor_series(x, a, b, c, d): #, d, e, f):\n#         return a + b * x + c * x**2 + d * x**3 #+ d * np.sin(e*x + f)# + d * x**3 \n    \n#     param, param_cov = curve_fit(taylor_series, list(x_axis[:40]) + list(x_axis[len(x_axis)-40:]), \n#                                                 list(series[:40]) + list(series[len(series)-40:]))\n    \n#     series /= taylor_series(x_axis, *param)\n    \n#     grad = np.gradient(series)\n#     cutoff = 1\n    \n#     x_start, x_end = np.where(grad == min(grad))[0][0], np.where(grad == max(grad))[0][0]\n    \n#     zero_to_t1, t1_to_t4, t4_to_end = series[40:x_start], series[x_start:x_end], series[x_end:len(series)-40]\n    \n#     index_in_transit = np.where(np.abs(series - np.mean(t1_to_t4)) < cutoff * np.std(t1_to_t4))[0]\n#     index_out_transit = np.where((np.abs(series - np.mean(zero_to_t1)) < cutoff * np.std(zero_to_t1)) | \\\n#                                (np.abs(series - np.mean(t4_to_end)) < cutoff * np.std(t4_to_end)))[0]\n    \n#     spectrum_in_transit = data[index_in_transit, :, :].sum(axis=2).sum(axis=0)\n#     spectrum_out_transit = data[index_out_transit, :, :].sum(axis=2).sum(axis=0)\n\n#     spectrum_error_in_transit = np.array([data[index_in_transit, i, :].flatten().std(axis=0) for i in range(data.shape[1])])\n#     spectrum_error_out_transit = np.array([data[index_in_transit, i, :].flatten().std(axis=0) for i in range(data.shape[1])])\n    \n#     # plt.plot(x_axis[index_out_transit], series[index_out_transit], 'b')\n#     # plt.plot(x_axis[index_in_transit], series[index_in_transit], 'r')\n\n    \n    \n#     if not os.path.exists(path_out + f\"AIRS_spectrum_in_transit/\"):\n#         os.mkdir(path_out + f\"AIRS_spectrum_in_transit/\")\n    \n#     np.save(path_out + f\"AIRS_spectrum_in_transit/data_{id}\",  spectrum_in_transit)\n#     np.save(path_out + f\"AIRS_spectrum_in_transit/error_{id}\",  spectrum_error_in_transit)\n\n    \n#     if not os.path.exists(path_out + f\"AIRS_spectrum_out_transit/\"):\n#         os.mkdir(path_out + f\"AIRS_spectrum_out_transit/\")\n    \n#     np.save(path_out + f\"AIRS_spectrum_out_transit/data_{id}\", spectrum_out_transit)\n#     np.save(path_out + f\"AIRS_spectrum_out_transit/error_{id}\",  spectrum_error_out_transit)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.optimize import curve_fit\n\nids = get_file_paths('/kaggle/input/long-run-neurips')\n\nfor n, id in enumerate(ids):\n    data = np.load(f'/kaggle/input/long-run-neurips/data_light_raw/AIRS_clean_train_{id}.npy')[0]\n    data[np.isnan(data)] = 0\n    \n    series = data.sum(axis=1).sum(axis=1)\n    series /= np.mean(series)\n    x_axis = np.arange(len(series))\n    \n    \n    def taylor_series(x, a, b, c, d): #, d, e, f):\n        return a + b * x + c * x**2 + d * x**3 #+ d * np.sin(e*x + f)# + d * x**3 \n    \n    param, param_cov = curve_fit(taylor_series, list(x_axis[:40]) + list(x_axis[len(x_axis)-40:]), \n                                                list(series[:40]) + list(series[len(series)-40:]))\n    \n    series /= taylor_series(x_axis, *param)\n    \n    grad = np.gradient(series)\n    cutoff = 1\n    \n    x_start, x_end = np.where(grad == min(grad))[0][0], np.where(grad == max(grad))[0][0]\n    \n    zero_to_t1, t1_to_t4, t4_to_end = series[40:x_start], series[x_start:x_end], series[x_end:len(series)-40]\n    \n    index_in_transit = np.where(np.abs(series - np.mean(t1_to_t4)) < cutoff * np.std(t1_to_t4))[0]\n    index_out_transit = np.where((np.abs(series - np.mean(zero_to_t1)) < cutoff * np.std(zero_to_t1)) | \\\n                               (np.abs(series - np.mean(t4_to_end)) < cutoff * np.std(t4_to_end)))[0]\n    \n    spectrum_in_transit = data[index_in_transit, :, :].sum(axis=2).mean(axis=0)\n    spectrum_out_transit = data[index_out_transit, :, :].sum(axis=2).mean(axis=0)\n    \n    if not os.path.exists(path_out + f\"{id}/\"):\n        os.mkdir(path_out + f\"{id}/\")\n\n    np.save(path_out + f\"{id}/full_data\",  data)\n\n\n    np.save(path_out + f\"{id}/in_transit_spectrum_data\",  spectrum_in_transit)\n    np.save(path_out + f\"{id}/in_transit_index\",  index_in_transit)\n\n\n    np.save(path_out + f\"{id}/out_transit_spectrum_data\", spectrum_out_transit)\n    np.save(path_out + f\"{id}/out_transit_index\",  index_out_transit)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T20:40:27.998466Z","iopub.execute_input":"2025-01-15T20:40:27.998976Z","iopub.status.idle":"2025-01-15T20:40:47.083918Z","shell.execute_reply.started":"2025-01-15T20:40:27.998939Z","shell.execute_reply":"2025-01-15T20:40:47.082053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}