{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-06T09:04:33.11963Z","iopub.execute_input":"2022-10-06T09:04:33.120005Z","iopub.status.idle":"2022-10-06T09:04:33.130764Z","shell.execute_reply.started":"2022-10-06T09:04:33.119923Z","shell.execute_reply":"2022-10-06T09:04:33.129873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## hdf5 dir:\n\n*  /kaggle/input/g2net-detecting-continuous-gravitational-waves/train/\n*  /kaggle/input/g2net-detecting-continuous-gravitational-waves/test/\n* /kaggle/input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv\n* /kaggle/input/g2net-detecting-continuous-gravitational-waves/train_labels.csv\n* /kaggle/input/g2net-detecting-continuous-gravitational-waves/train/de9b07e8c.hdf5","metadata":{}},{"cell_type":"code","source":"\nimport os\n# change file path to working directory\n# w = os.chdir('../input/g2net-detecting-continuous-gravitational-waves/train/')\n# # check the current working directory\n# wd = os.getcwd()\n# print(\"Working directory\",wd)\n# with pd.HDFStore('../input/g2net-detecting-continuous-gravitational-waves/train/de9b07e8c.hdf5',\"r\") as train:\n#     df = train.get(\"de9b07e8c\")\n    \n# data = pd.read_hdf(\"../input/g2net-detecting-continuous-gravitational-waves/train/de9b07e8c.hdf5\",\"de9b07e8c\")\n# print('Dataset size: ',data.shape)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-10-06T09:04:33.131875Z","iopub.execute_input":"2022-10-06T09:04:33.132204Z","iopub.status.idle":"2022-10-06T09:04:33.148987Z","shell.execute_reply.started":"2022-10-06T09:04:33.132172Z","shell.execute_reply":"2022-10-06T09:04:33.14804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import h5py as h5\nimport matplotlib.pyplot as plt\n\n# Read H5 File\nf = h5.File(\"../input/g2net-detecting-continuous-gravitational-waves/train/004f23b2d.hdf5\",\"r\")\n\ndatasetNames = [n for n in f.keys()]\nfor n in datasetNames:\n    print(n)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:04:53.125366Z","iopub.execute_input":"2022-10-06T09:04:53.126023Z","iopub.status.idle":"2022-10-06T09:04:53.14363Z","shell.execute_reply.started":"2022-10-06T09:04:53.125984Z","shell.execute_reply":"2022-10-06T09:04:53.142587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(f.keys())","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:04:57.856889Z","iopub.execute_input":"2022-10-06T09:04:57.858068Z","iopub.status.idle":"2022-10-06T09:04:57.867092Z","shell.execute_reply.started":"2022-10-06T09:04:57.858025Z","shell.execute_reply":"2022-10-06T09:04:57.866145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(f)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:04:57.868908Z","iopub.execute_input":"2022-10-06T09:04:57.869224Z","iopub.status.idle":"2022-10-06T09:04:57.882923Z","shell.execute_reply.started":"2022-10-06T09:04:57.869195Z","shell.execute_reply":"2022-10-06T09:04:57.88208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv(\"/kaggle/input/g2net-detecting-continuous-gravitational-waves/train_labels.csv\")\nlabels.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:04:57.88429Z","iopub.execute_input":"2022-10-06T09:04:57.884876Z","iopub.status.idle":"2022-10-06T09:04:57.906364Z","shell.execute_reply.started":"2022-10-06T09:04:57.88484Z","shell.execute_reply":"2022-10-06T09:04:57.905531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import h5py\n\nf1 = f\n\nf1.keys()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:05:10.197883Z","iopub.execute_input":"2022-10-06T09:05:10.198335Z","iopub.status.idle":"2022-10-06T09:05:10.205955Z","shell.execute_reply.started":"2022-10-06T09:05:10.198297Z","shell.execute_reply":"2022-10-06T09:05:10.205003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid = f1['004f23b2d']\ngrid.keys()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:05:14.086428Z","iopub.execute_input":"2022-10-06T09:05:14.086873Z","iopub.status.idle":"2022-10-06T09:05:14.100516Z","shell.execute_reply.started":"2022-10-06T09:05:14.086837Z","shell.execute_reply":"2022-10-06T09:05:14.09932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observe that there are a lot of values ('H1', 'L1', 'frequency_Hz') in this data file. Let's take a brief look at them.","metadata":{}},{"cell_type":"markdown","source":"## H1 data","metadata":{}},{"cell_type":"code","source":"print(\"H1 data : {}\".format(grid['H1']))\nprint(\"H1 data attributes {}\".format(list(grid['H1'].keys())))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:05:14.102323Z","iopub.execute_input":"2022-10-06T09:05:14.102655Z","iopub.status.idle":"2022-10-06T09:05:14.115575Z","shell.execute_reply.started":"2022-10-06T09:05:14.102623Z","shell.execute_reply":"2022-10-06T09:05:14.114503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# print(\"SFTS Contents: {}\".format(list(grid['H1']['SFTs'])))\nprint(\"SFTS: {}\".format(grid['H1']['SFTs']))\nprint(\"timestamps_GPS: {}\".format(grid['H1']['timestamps_GPS']))\n","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:05:14.117033Z","iopub.execute_input":"2022-10-06T09:05:14.117463Z","iopub.status.idle":"2022-10-06T09:05:14.13405Z","shell.execute_reply.started":"2022-10-06T09:05:14.117426Z","shell.execute_reply":"2022-10-06T09:05:14.132837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The short-time Fourier transform (STFT) is used to analyze how the frequency content of a nonstationary signal changes over time. ","metadata":{}},{"cell_type":"markdown","source":"# L1 data","metadata":{}},{"cell_type":"code","source":"print(\"L1 data : {}\".format(grid['L1']))\nprint(\"L1 data attributes {}\".format(list(grid['L1'].attrs)))\nprint(\"L1 data keys {}\".format(list(grid['L1'].keys())))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:05:14.137175Z","iopub.execute_input":"2022-10-06T09:05:14.137585Z","iopub.status.idle":"2022-10-06T09:05:14.148436Z","shell.execute_reply.started":"2022-10-06T09:05:14.137547Z","shell.execute_reply":"2022-10-06T09:05:14.147101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"SFTS: {}\".format(grid['L1']['SFTs']))\nprint(\"timestamps_GPS: {}\".format(grid['L1']['timestamps_GPS']))\n","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:05:14.150152Z","iopub.execute_input":"2022-10-06T09:05:14.151508Z","iopub.status.idle":"2022-10-06T09:05:14.162308Z","shell.execute_reply.started":"2022-10-06T09:05:14.151464Z","shell.execute_reply":"2022-10-06T09:05:14.161117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Frequency_Hz data","metadata":{}},{"cell_type":"code","source":"print(\"Frequency data: {}\".format(grid[\"frequency_Hz\"]))\nprint(\"Frequency data attributes: {}\".format(list(grid[\"frequency_Hz\"].attrs)))\nprint(\"Frequency keys: {}\".format(grid[\"frequency_Hz\"].shape))\n","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:05:14.164Z","iopub.execute_input":"2022-10-06T09:05:14.164526Z","iopub.status.idle":"2022-10-06T09:05:14.176806Z","shell.execute_reply.started":"2022-10-06T09:05:14.164476Z","shell.execute_reply":"2022-10-06T09:05:14.175704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data transformation","metadata":{}},{"cell_type":"markdown","source":"Let's convert all the data in training directory into a large dataframe and explore the dataset","metadata":{}},{"cell_type":"code","source":"import glob\npath = os.getcwd()\nhdf5_files = glob.glob(os.path.join(path,\"*.hdf5\"))\nfor f in hdf5_files:\n    with h5py.File(f,\"r\") as hf:\n        grid = hf[str(list(hf.keys())[0])]\n        data = list(map(lambda x: grid[x],grid.keys()))\n        \n        print(data)\n#     print(list(hf.keys())[0])\n    \n    \n    \n#     print('Location: ', f)\n#     print('File Name: ', f.split(\"\\\\\")[-1])\n#     print('Content:')\n#     display(df)\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:06:10.052534Z","iopub.execute_input":"2022-10-06T09:06:10.052929Z","iopub.status.idle":"2022-10-06T09:06:10.061175Z","shell.execute_reply.started":"2022-10-06T09:06:10.052893Z","shell.execute_reply":"2022-10-06T09:06:10.060146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Work update:\n\nHey folks, The submission and model building is pending. Work in progress. This Jupyter notebook is a short data exploration series.\n\nStay tuned. More content to be uploaded soon. ","metadata":{}}]}