{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2Net:Noise reduction part\n## Reduction of non-stationary noise for test data","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## Import","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport h5py\nimport torch","metadata":{"execution":{"iopub.status.busy":"2023-01-03T23:59:14.895527Z","iopub.execute_input":"2023-01-03T23:59:14.896621Z","iopub.status.idle":"2023-01-03T23:59:16.714679Z","shell.execute_reply.started":"2023-01-03T23:59:14.896521Z","shell.execute_reply":"2023-01-03T23:59:16.713802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"# Noise reduction method\ndef noise_reduction(p):\n    return (p-np.median(p, axis=0))/np.std(p, axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T23:59:16.716509Z","iopub.execute_input":"2023-01-03T23:59:16.71756Z","iopub.status.idle":"2023-01-03T23:59:16.724117Z","shell.execute_reply.started":"2023-01-03T23:59:16.717526Z","shell.execute_reply":"2023-01-03T23:59:16.721734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Code modified from JUN KODA's [Basic spectrogram image classification](https://www.kaggle.com/code/junkoda/basic-spectrogram-image-classification).","metadata":{}},{"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    def __init__(self, data_type, df):\n        self.data_type = data_type\n        self.df = df\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        r = self.df.iloc[i]\n        y = np.float32(r.target)\n        file_id = r.id\n\n        img = np.empty((2, 360, 128), dtype=np.float32)\n        img_div = np.empty((2, 360, 128), dtype=np.float32)\n\n        filename = '%s/%s/%s.hdf5' % (di, self.data_type, file_id)        \n\n        with h5py.File(filename, 'r') as f:\n            g = f[file_id]\n\n            for ch, s in enumerate(['H1', 'L1']):\n                a = g[s]['SFTs'][:, :4096] * 1e22\n                p = a.real**2 + a.imag**2\n                \n                p_div = noise_reduction(p)# Reduction of non-stationary noise\n                p = p/np.mean(p)          # original\n                \n                \n                p = np.mean(p.reshape(360, 128, 32), axis=2)\n                p_div = np.mean(p_div.reshape(360, 128, 32), axis=2)\n\n                \n                img[ch] = p\n                img_div[ch] = p_div\n\n                \n        return img, y, file_id, img_div","metadata":{"execution":{"iopub.status.busy":"2023-01-03T23:59:16.725532Z","iopub.execute_input":"2023-01-03T23:59:16.72657Z","iopub.status.idle":"2023-01-03T23:59:16.741249Z","shell.execute_reply.started":"2023-01-03T23:59:16.726527Z","shell.execute_reply":"2023-01-03T23:59:16.739921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Apply noise reduction method to data","metadata":{}},{"cell_type":"code","source":"def plot_img(img, no_subplot, title):\n    plt.subplot(2,2,no_subplot)\n    plt.title(title)\n    plt.xlabel('time')\n    plt.ylabel('frequency')\n    plt.imshow(img)  \n    plt.colorbar()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T23:59:16.742874Z","iopub.execute_input":"2023-01-03T23:59:16.743966Z","iopub.status.idle":"2023-01-03T23:59:16.750682Z","shell.execute_reply.started":"2023-01-03T23:59:16.743922Z","shell.execute_reply":"2023-01-03T23:59:16.749794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test data","metadata":{}},{"cell_type":"code","source":"batch_size = 32\nnum_workers = 2\ndi =  '../input/g2net-detecting-continuous-gravitational-waves'\n\nsubmit = pd.read_csv(di + '/sample_submission.csv')\ndataset_test = Dataset('test', submit)\nloader_test = torch.utils.data.DataLoader(dataset_test, batch_size=batch_size,\n                                            num_workers=num_workers, pin_memory=True)\n\nfor i in [50, 102, 502, 709, 2005, 63, 65]:\n\n    img, y, file_id, img_div = dataset_test[i]\n    plt.figure(figsize=(6, 10))\n    plt.suptitle(file_id)\n    plot_img(img[0, 0:360], 1, 'H1 original')\n    plot_img(img[1, 0:360], 3, 'L1 original')\n    plot_img(img_div[0, 0:360], 2, 'H1 noise reduction')\n    plot_img(img_div[1, 0:360], 4, 'L1 noise reduction')\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T23:59:16.753384Z","iopub.execute_input":"2023-01-03T23:59:16.754573Z","iopub.status.idle":"2023-01-03T23:59:25.7586Z","shell.execute_reply.started":"2023-01-03T23:59:16.75453Z","shell.execute_reply":"2023-01-03T23:59:25.757423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(di + '/train_labels.csv')\ndataset_train = Dataset('train', df)\nloader_train = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size,\n                                            num_workers=num_workers, pin_memory=True)\n\nfor i in [2, 100, 205, 344, 455, 576, 600]:\n\n    img, y, file_id, img_div = dataset_train[i]\n    plt.figure(figsize=(6, 10))\n    plt.suptitle(file_id)\n    plot_img(img[0, 0:360], 1, 'H1 original')\n    plot_img(img[1, 0:360], 3, 'L1 original')\n    plot_img(img_div[0, 0:360], 2, 'H1 noise reduction')\n    plot_img(img_div[1, 0:360], 4, 'L1 noise reduction')\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T23:59:25.760336Z","iopub.execute_input":"2023-01-03T23:59:25.760729Z","iopub.status.idle":"2023-01-03T23:59:34.353675Z","shell.execute_reply.started":"2023-01-03T23:59:25.760648Z","shell.execute_reply":"2023-01-03T23:59:34.352517Z"},"trusted":true},"execution_count":null,"outputs":[]}]}