{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport h5py\nimport cmath\nimport matplotlib.pyplot as plt\nimport re, glob, copy, collections\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader","metadata":{"execution":{"iopub.status.busy":"2022-12-11T05:53:58.623664Z","iopub.execute_input":"2022-12-11T05:53:58.624176Z","iopub.status.idle":"2022-12-11T05:54:00.463252Z","shell.execute_reply.started":"2022-12-11T05:53:58.62407Z","shell.execute_reply":"2022-12-11T05:54:00.462209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class datasets(Dataset):\n    def __init__(self,train=True):\n        super().__init__()\n        self.train = train\n        self.base_dir = '/kaggle/input/g2net-detecting-continuous-gravitational-waves'\n        self.train_dir = f'{self.base_dir}/train'\n        self.test_dir =  f'{self.base_dir}/test'\n        if self.train:\n            self.df = pd.read_csv(f\"{self.base_dir}/train_labels.csv\")\n        else:\n            self.df = pd.read_csv(f\"/kaggle/input/make-sub4fold/test.csv\")\n\n    def __len__(self):\n        return len(self.df)\n    \n    def mono_to_color(self, X, eps=1e-30):\n        X = (X - X.mean()) / (X.std() + eps)\n\n        _min, _max = X.min(), X.max()\n\n        if (_max - _min) > eps:\n            V = np.clip(X, _min, _max)\n            V = 255 * (V - _min) / (_max - _min)\n            V = V.astype(np.uint8)\n        else:\n            V = np.zeros_like(X, dtype=np.uint8)\n\n        return V\n        \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        if self.train:\n            f = h5py.File(f'{self.train_dir}/{row.id}.hdf5', 'r')\n        else:\n            f = h5py.File(f'{self.test_dir}/{row.id}.hdf5', 'r')\n        \n        h1_time = f[row.id]['H1']['timestamps_GPS'][:]\n        l1_time = f[row.id]['L1']['timestamps_GPS'][:]\n        h1_img_real = f[row.id]['H1']['SFTs'][:,:].real*1e22\n        h1_img_imag = f[row.id]['H1']['SFTs'][:,:].imag*1e22\n        h1_img = np.sqrt(h1_img_real**2 + h1_img_imag**2)\n        l1_img_real = f[row.id]['L1']['SFTs'][:,:].real*1e22\n        l1_img_imag = f[row.id]['L1']['SFTs'][:,:].imag*1e22\n        l1_img = np.sqrt(l1_img_real**2 + l1_img_imag**2)\n        \n        freq = f[row.id]['frequency_Hz'][:]\n        if self.train:\n            labels = row.target\n            return row.id, h1_img, l1_img, freq, h1_time, l1_time, labels\n        else:\n            return row.id, h1_img, l1_img, freq, h1_time, l1_time","metadata":{"execution":{"iopub.status.busy":"2022-12-11T06:00:13.672198Z","iopub.execute_input":"2022-12-11T06:00:13.672689Z","iopub.status.idle":"2022-12-11T06:00:13.716904Z","shell.execute_reply.started":"2022-12-11T06:00:13.672653Z","shell.execute_reply":"2022-12-11T06:00:13.715739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = datasets(train=True)\nids = []\nl1_len = []\nl1_max = []\nl1_min = []\nl1_mean = []\n\nh1_len = []\nh1_max = []\nh1_min = []\nh1_mean = []\n\nfreq_max = []\nfreq_min = []\nfreq_std = []\nfreq_median = []\n\nh1_start = []\nh1_duration = []\n\nl1_start = []\nl1_duration = []\nl1_diff = []\n\nh1_start = []\nh1_duration = []\nh1_diff = []\n\nlabels = []\n\nfor idx, (id, h1_img, l1_img, freq, h1_t, l1_t, label) in enumerate(dataset):\n    ids.append(id)\n    l1_len.append(len(l1_t))\n    l1_max.append(l1_img.max())\n    l1_min.append(l1_img.min())\n    l1_mean.append(l1_img.mean())\n    \n    h1_len.append(len(h1_t))\n    h1_max.append(h1_img.max())\n    h1_min.append(h1_img.min())\n    h1_mean.append(h1_img.mean())\n    \n    freq_max.append(freq.max())\n    freq_min.append(freq.min())\n    freq_std.append(freq.std())\n    freq_median.append(np.median(freq))\n    \n    l1_start.append(l1_t.min())\n    l1_duration.append(l1_t.max() - l1_t.min())\n    l1_diff.append(np.diff(l1_t).mean())\n    \n    h1_start.append(h1_t.min())\n    h1_duration.append(h1_t.max() - h1_t.min())\n    h1_diff.append(np.diff(h1_t).mean())\n    \n    labels.append(label)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T06:00:13.911051Z","iopub.execute_input":"2022-12-11T06:00:13.911506Z","iopub.status.idle":"2022-12-11T06:02:57.170745Z","shell.execute_reply.started":"2022-12-11T06:00:13.911466Z","shell.execute_reply":"2022-12-11T06:02:57.169551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_array = [\n    ids,labels,\n    l1_len,l1_max,l1_min,l1_mean,\n    h1_len,h1_max,h1_min,h1_mean,\n    freq_max,freq_min,freq_std,freq_median, \n    l1_start, l1_duration, l1_diff,\n    h1_start, h1_duration, h1_diff\n]\n\ninput_columns = [\n    \"ids\",\"label\",\n    \"l1_len\",\"l1_max\",\"l1_min\",\"l1_mean\",\n    \"h1_len\",\"h1_max\",\"h1_min\",\"h1_mean\",\n    \"freq_max\",\"freq_min\",\"freq_std\",\"freq_median\",\n    \"l1_start\", \"l1_duration\", \"l1_diff\",\n    \"h1_start\", \"h1_duration\", \"h1_diff\"\n]\npd.DataFrame(input_array,index=input_columns).T.to_csv(\"train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-11T06:02:57.17299Z","iopub.execute_input":"2022-12-11T06:02:57.174108Z","iopub.status.idle":"2022-12-11T06:02:57.217767Z","shell.execute_reply.started":"2022-12-11T06:02:57.174066Z","shell.execute_reply":"2022-12-11T06:02:57.216801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = datasets(train=False)\nids = []\nl1_len = []\nl1_max = []\nl1_min = []\nl1_mean = []\n\nh1_len = []\nh1_max = []\nh1_min = []\nh1_mean = []\n\nfreq_max = []\nfreq_min = []\nfreq_std = []\nfreq_median = []\n\nh1_start = []\nh1_duration = []\n\nl1_start = []\nl1_duration = []\nl1_diff = []\n\nh1_start = []\nh1_duration = []\nh1_diff = []\n\n\nfor idx, (id, h1_img, l1_img, freq, h1_t, l1_t) in enumerate(dataset):\n    ids.append(id)\n    l1_len.append(len(l1_t))\n    l1_max.append(l1_img.max())\n    l1_min.append(l1_img.min())\n    l1_mean.append(l1_img.mean())\n    \n    h1_len.append(len(h1_t))\n    h1_max.append(h1_img.max())\n    h1_min.append(h1_img.min())\n    h1_mean.append(h1_img.mean())\n    \n    freq_max.append(freq.max())\n    freq_min.append(freq.min())\n    freq_std.append(freq.std())\n    freq_median.append(np.median(freq))\n    \n    l1_start.append(l1_t.min())\n    l1_duration.append(l1_t.max() - l1_t.min())\n    l1_diff.append(np.diff(l1_t).mean())\n    \n    h1_start.append(h1_t.min())\n    h1_duration.append(h1_t.max() - h1_t.min())\n    h1_diff.append(np.diff(h1_t).mean())","metadata":{"execution":{"iopub.status.busy":"2022-12-11T05:54:01.20862Z","iopub.execute_input":"2022-12-11T05:54:01.20932Z","iopub.status.idle":"2022-12-11T05:54:27.091466Z","shell.execute_reply.started":"2022-12-11T05:54:01.209278Z","shell.execute_reply":"2022-12-11T05:54:27.089539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_array = [\n    ids,\n    l1_len,l1_max,l1_min,l1_mean,\n    h1_len,h1_max,h1_min,h1_mean,\n    freq_max,freq_min,freq_std,freq_median, \n    l1_start, l1_duration, l1_diff,\n    h1_start, h1_duration, h1_diff\n]\n\ninput_columns = [\n    \"ids\",\n    \"l1_len\",\"l1_max\",\"l1_min\",\"l1_mean\",\n    \"h1_len\",\"h1_max\",\"h1_min\",\"h1_mean\",\n    \"freq_max\",\"freq_min\",\"freq_std\",\"freq_median\",\n    \"l1_start\", \"l1_duration\", \"l1_diff\",\n    \"h1_start\", \"h1_duration\", \"h1_diff\"\n]\npd.DataFrame(input_array,index=input_columns).T.to_csv(\"test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-11T05:54:27.092351Z","iopub.status.idle":"2022-12-11T05:54:27.092771Z","shell.execute_reply.started":"2022-12-11T05:54:27.09257Z","shell.execute_reply":"2022-12-11T05:54:27.09259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}