{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-25T09:05:44.227902Z","iopub.execute_input":"2022-12-25T09:05:44.228414Z","iopub.status.idle":"2022-12-25T09:05:49.703746Z","shell.execute_reply.started":"2022-12-25T09:05:44.228312Z","shell.execute_reply":"2022-12-25T09:05:49.702683Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:05:49.70576Z","iopub.execute_input":"2022-12-25T09:05:49.706364Z","iopub.status.idle":"2022-12-25T09:05:50.328406Z","shell.execute_reply.started":"2022-12-25T09:05:49.706323Z","shell.execute_reply":"2022-12-25T09:05:50.327179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '../input/g2net-detecting-continuous-gravitational-waves'\nos.path.isdir(ROOT_DIR)","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:09:18.615209Z","iopub.execute_input":"2022-12-25T09:09:18.615705Z","iopub.status.idle":"2022-12-25T09:09:18.627234Z","shell.execute_reply.started":"2022-12-25T09:09:18.615665Z","shell.execute_reply":"2022-12-25T09:09:18.626377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train_labels.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:09:32.562975Z","iopub.execute_input":"2022-12-25T09:09:32.564036Z","iopub.status.idle":"2022-12-25T09:09:32.593131Z","shell.execute_reply.started":"2022-12-25T09:09:32.563983Z","shell.execute_reply":"2022-12-25T09:09:32.592391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nfrom pathlib import Path\nimport h5py","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:09:48.658975Z","iopub.execute_input":"2022-12-25T09:09:48.659391Z","iopub.status.idle":"2022-12-25T09:09:48.827575Z","shell.execute_reply.started":"2022-12-25T09:09:48.659358Z","shell.execute_reply":"2022-12-25T09:09:48.826326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = glob(f\"{ROOT_DIR}/train/*.hdf5\")\ntest_files = glob(f\"{ROOT_DIR}/test/*.hdf5\")\ntrain_files[:5]","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:10:06.920064Z","iopub.execute_input":"2022-12-25T09:10:06.92047Z","iopub.status.idle":"2022-12-25T09:10:06.9611Z","shell.execute_reply.started":"2022-12-25T09:10:06.920438Z","shell.execute_reply":"2022-12-25T09:10:06.960327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from random import sample\ndef remove_n_random_items(lst, n):\n    to_delete = set(sample(range(len(lst)), n))\n\n    return [\n        item for index, item in enumerate(lst)\n        if not index in to_delete\n    ]\n#trimmed=remove_n_random_items(times['H1'], 200)\n# print(len(trimmed))\n# # 👇️ ['bobby', 'hadz', '.', 'com', 'c']\n# print(remove_n_random_items(times['H1'], 420))\n# import copy\n# #new_list = copy.deepcopy(old_list)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Utility to read hdf5 file\ndef read_data(file: Path):\n    with h5py.File(file, \"r\") as f:\n        file=Path(file)\n        filename = file.stem\n        f = f[filename]\n        h1 = f[\"H1\"]\n        l1 = f[\"L1\"]\n        freq_hz = list(f[\"frequency_Hz\"])\n        \n        h1_stft = h1[\"SFTs\"][()]\n        h1_timestamp = h1[\"timestamps_GPS\"][()]\n        # H2 data\n        l1_stft = l1[\"SFTs\"][()]\n        l1_timestamp = l1[\"timestamps_GPS\"][()]\n        \n        return {\n            \"H1\": [h1_stft, h1_timestamp],\n            \"L1\": [l1_stft, l1_timestamp],\n            \"freq_hz\": freq_hz\n        }\ndef read_data2(file: Path):\n    with h5py.File(file, \"r\") as f:\n        file=Path(file)\n        filename = file.stem\n        f = f[filename]\n        h1 = f[\"H1\"]\n        l1 = f[\"L1\"]\n        freq_hz = list(f[\"frequency_Hz\"])\n        \n        h1_stft = h1[\"SFTs\"][()]\n        h1_timestamp = h1[\"timestamps_GPS\"][()]\n        # H2 data\n        l1_stft = l1[\"SFTs\"][()]\n        l1_timestamp = l1[\"timestamps_GPS\"][()]\n        \n        return {\n            h1_stft, h1_timestamp ,l1_stft, l1_timestamp\n        }","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:21:02.310848Z","iopub.execute_input":"2022-12-25T09:21:02.311453Z","iopub.status.idle":"2022-12-25T09:21:02.324617Z","shell.execute_reply.started":"2022-12-25T09:21:02.311403Z","shell.execute_reply":"2022-12-25T09:21:02.32337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_h5py_file_slice(frequency,fourier_data,timestamps,i,path=\"/kaggle/working/noise/\",name=\"Gausian_Noise\"):\n#frequency,fourier_data,timestamps,path=\"/kaggle/working/noise/\",name=\"Gausian_Noise\",i        \n        \n        print(freqs.shape)\n        slice_min=freqs.shape[0]//2-180\n        slice_max=freqs.shape[0]//2+180\n        file_path=path+name\n        f = h5py.File(f'{file_path}{i}.hdf5','w')\n        g0 = f.create_group(f\"{name}{i}\")\n        g1 = g0.create_group(\"H1\")\n        g2 = g0.create_group(\"L1\")\n        print(frequency.shape[0])\n        frequency= frequency[slice_min:slice_max]\n        print(frequency.shape[0])\n        dset = g0.create_dataset(\"frequency_Hz\", data=frequency)\n        temp=fourier_data['H1']\n        temp= temp[slice_min:slice_max,:]\n        print(temp.shape)\n        temp=temp[0:360,:]\n        if temp.shape[1]>4400:\n            temp=temp[0:360,0:4400]\n        print(temp.shape)\n        g1.create_dataset(\"SFTs\",data=temp)\n        g1.create_dataset(\"timestamps_GPS\",data=timestamps['H1'])\n        temp=fourier_data['L1']\n        temp= temp[slice_min:slice_max,:]\n        temp=temp[0:360,:]\n        if temp.shape[1]>4400:\n            temp=temp[0:360,0:4400]\n        g2.create_dataset(\"SFTs\",data=temp)\n        g2.create_dataset(\"timestamps_GPS\",data=timestamps['L1'])\n        f.close()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files[:2]","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:17:35.932384Z","iopub.execute_input":"2022-12-25T09:17:35.933359Z","iopub.status.idle":"2022-12-25T09:17:35.940619Z","shell.execute_reply.started":"2022-12-25T09:17:35.933317Z","shell.execute_reply":"2022-12-25T09:17:35.939338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_file=read_data(train_files[1])\none_file['H1'][1]","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:27:10.769873Z","iopub.execute_input":"2022-12-25T09:27:10.770345Z","iopub.status.idle":"2022-12-25T09:27:11.037825Z","shell.execute_reply.started":"2022-12-25T09:27:10.770303Z","shell.execute_reply":"2022-12-25T09:27:11.036363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lenh1=[]\nlenl1=[]\nfor elem in train_files:\n    file_dict=read_data(elem)\n    h1_stft=file_dict['H1'][0]\n    h1_timestamp =file_dict['H1'][1]\n    \n    l1_stft=file_dict['L1'][0]\n    l1_timestamp =file_dict['L1'][1]\n    freq_hz=file_dict['freq_hz']\n    len_timeh1=len(h1_timestamp)\n    if len_timeh1>4320:\n        lenh1.append(len_timeh1)\n    len_timel1=len(l1_timestamp)\n    if len_timel1>4320:\n        lenl1.append(len_timel1)\n    if (len_timel1>4320)\n    #print(elem)","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:42:01.625948Z","iopub.execute_input":"2022-12-25T09:42:01.626545Z","iopub.status.idle":"2022-12-25T09:44:32.929031Z","shell.execute_reply.started":"2022-12-25T09:42:01.626503Z","shell.execute_reply":"2022-12-25T09:44:32.927961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:44:32.931121Z","iopub.execute_input":"2022-12-25T09:44:32.931861Z","iopub.status.idle":"2022-12-25T09:44:32.937154Z","shell.execute_reply.started":"2022-12-25T09:44:32.931818Z","shell.execute_reply":"2022-12-25T09:44:32.936037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:44:32.938611Z","iopub.execute_input":"2022-12-25T09:44:32.939055Z","iopub.status.idle":"2022-12-25T09:44:32.950148Z","shell.execute_reply.started":"2022-12-25T09:44:32.93902Z","shell.execute_reply":"2022-12-25T09:44:32.94918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(lenh1,bins=100)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:44:32.952849Z","iopub.execute_input":"2022-12-25T09:44:32.953757Z","iopub.status.idle":"2022-12-25T09:44:33.674929Z","shell.execute_reply.started":"2022-12-25T09:44:32.95372Z","shell.execute_reply":"2022-12-25T09:44:33.67374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(lenh1)","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:45:45.915658Z","iopub.execute_input":"2022-12-25T09:45:45.916116Z","iopub.status.idle":"2022-12-25T09:45:45.925188Z","shell.execute_reply.started":"2022-12-25T09:45:45.916081Z","shell.execute_reply":"2022-12-25T09:45:45.924014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(lenl1,bins=100)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:44:33.677009Z","iopub.execute_input":"2022-12-25T09:44:33.677888Z","iopub.status.idle":"2022-12-25T09:44:34.033452Z","shell.execute_reply.started":"2022-12-25T09:44:33.677837Z","shell.execute_reply":"2022-12-25T09:44:34.032167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(lenl1)","metadata":{"execution":{"iopub.status.busy":"2022-12-25T09:46:02.846443Z","iopub.execute_input":"2022-12-25T09:46:02.846988Z","iopub.status.idle":"2022-12-25T09:46:02.854726Z","shell.execute_reply.started":"2022-12-25T09:46:02.84695Z","shell.execute_reply":"2022-12-25T09:46:02.853257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}