{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Use timm pretrained image model\n! pip3 install timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-23T07:09:20.32421Z","iopub.execute_input":"2022-11-23T07:09:20.324644Z","iopub.status.idle":"2022-11-23T07:09:34.476905Z","shell.execute_reply.started":"2022-11-23T07:09:20.324557Z","shell.execute_reply":"2022-11-23T07:09:34.475625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport time\nimport h5py\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport random\nimport gc,os,sys,shutil \nfrom torch.nn import DataParallel\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom timm.scheduler import CosineLRScheduler\nimport torchvision.transforms as transforms\nimport torchaudio\nimport torchvision.transforms as TF\nfrom transformers.file_utils import is_torch_tpu_available\nimport scipy\nfrom scipy.fftpack import fftshift\nfrom scipy import ndimage\nimport cv2\nfrom sklearn.feature_extraction import image\nimport pywt\nfrom scipy.stats import norm\ngpus = [0,1]\ndevice = torch.device('cuda') if torch.cuda.is_available() else 'cpu'\ncriterion = nn.BCEWithLogitsLoss()\n\n# Train metadata\ndi = '/kaggle/input/g2net-detecting-continuous-gravitational-waves'\ndf = pd.read_csv(di + '/train_labels.csv')\ndf = df[df.target >= 0]  # Remove 3 unknowns (target = -1)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:34.480895Z","iopub.execute_input":"2022-11-23T07:09:34.481218Z","iopub.status.idle":"2022-11-23T07:09:41.094393Z","shell.execute_reply.started":"2022-11-23T07:09:34.481187Z","shell.execute_reply":"2022-11-23T07:09:41.093085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Params","metadata":{}},{"cell_type":"code","source":"# params\nPRETRAIN = True\nSINGLE_MODEL = True\nIMG_SIZE = None\nIF_GPU = True\n#MODEL_NAME = 'tf_efficientnet_b4_ns'\n\n#MODEL_NAME = 'tf_efficientnet_b5_ns'\n\nmodel_name = 'tf_efficientnet_b7_ns'\n# model_name = 'tf_efficientnetv2_b0'\nsubmit = pd.read_csv(di + '/sample_submission.csv')\n# MODEL_NAME = 'eca_nfnet_l3'\n# MODEL_NAME = 'levit_192'\n# IMG_SIZE = 224 # resize the images for a specific image models\n\n# shapes to resize the images\nSHAPE_1 = (128,32)\nSHAPE_2 = (360,1)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:41.096322Z","iopub.execute_input":"2022-11-23T07:09:41.096669Z","iopub.status.idle":"2022-11-23T07:09:41.115221Z","shell.execute_reply.started":"2022-11-23T07:09:41.096633Z","shell.execute_reply":"2022-11-23T07:09:41.114339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:41.116799Z","iopub.execute_input":"2022-11-23T07:09:41.11719Z","iopub.status.idle":"2022-11-23T07:09:41.126523Z","shell.execute_reply.started":"2022-11-23T07:09:41.117155Z","shell.execute_reply":"2022-11-23T07:09:41.125101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":" def laplace_sharpen(input_image, c):\n        input_image_cp = np.copy(input_image)  # 输入图像的副本\n \n     # 拉普拉斯滤波器\n        laplace_filter = np.array([\n        [1, 1, 1],\n        [1, -8, 1],\n        [1, 1, 1],\n     ])\n \n        input_image_cp = np.pad(input_image_cp, (1, 1), mode='constant', constant_values=0)  # 填充输入图像\n\n        m, n = input_image_cp.shape  # 填充后的输入图像的尺寸\n\n        output_image = np.copy(input_image_cp)  # 输出图像\n\n        for i in range(1, m - 1):\n             for j in range(1, n - 1):\n                R = np.sum(laplace_filter * input_image_cp[i - 1:i + 2, j - 1:j + 2])  # 拉普拉斯滤波器响应\n\n                output_image[i, j] = input_image_cp[i, j] + c * R\n\n        output_image = output_image[1:m - 1, 1:n - 1]  # 裁剪\n\n        return output_image","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:41.131128Z","iopub.execute_input":"2022-11-23T07:09:41.132892Z","iopub.status.idle":"2022-11-23T07:09:41.142033Z","shell.execute_reply.started":"2022-11-23T07:09:41.132866Z","shell.execute_reply":"2022-11-23T07:09:41.14116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_hough_space_fom_image(img, shape = (360, 128), val = 1):\n    hough_space = np.zeros(shape)\n    for i, row in enumerate(img):\n        for j, pixel in enumerate(row):   \n            if pixel != val : continue\n        hough_space = add_to_hough_space_polar((i,j), hough_space)\n    return hough_space\ndef add_to_hough_space_polar(p, feature_space):\n    space = np.linspace(0, np.pi, len(feature_space))\n    d_max = len(feature_space[0]) / 2\n    for i in range(len(space)):\n        theta = space[i]\n        d = int(p[0] * np.sin(theta) + p[1] * np.cos(theta)) + d_max\n        if (d >= d_max * 2) : continue\n        feature_space[i, int(d)] += 1\n    return feature_space","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:41.144702Z","iopub.execute_input":"2022-11-23T07:09:41.145793Z","iopub.status.idle":"2022-11-23T07:09:41.156393Z","shell.execute_reply.started":"2022-11-23T07:09:41.145757Z","shell.execute_reply":"2022-11-23T07:09:41.155596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize(X):\n    X = (X[..., None].view(X.real.dtype) ** 2).sum(-1)\n    POS = int(X.size * 0.99903)\n    EXP = norm.ppf((POS + 0.4) / (X.size + 0.215))\n    scale = np.partition(X.flatten(), POS, -1)[POS]\n    X /= scale / EXP.astype(scale.dtype) ** 2\n    return X","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:41.158248Z","iopub.execute_input":"2022-11-23T07:09:41.158526Z","iopub.status.idle":"2022-11-23T07:09:41.169035Z","shell.execute_reply.started":"2022-11-23T07:09:41.158502Z","shell.execute_reply":"2022-11-23T07:09:41.168138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms_time_mask = nn.Sequential(\n                torchaudio.transforms.TimeMasking(time_mask_param=10),\n            )\n\ntransforms_freq_mask = nn.Sequential(\n                torchaudio.transforms.FrequencyMasking(freq_mask_param=10),\n            )\ntransforms_MelSpec = nn.Sequential(\n                torchaudio.transforms.TimeStretch(fixed_rate=30,n_freq=360)\n#                 torchaudio.transforms.MelSpectrogram(sample_rate = 16000, n_fft = 360, normalized=True,n_mels=32),\n            )\n\nflip_rate = 0.0 # probability of applying the horizontal flip and vertical flip \nfre_shift_rate = 0.0 # probability of applying the vertical shift\ntime_mask_num = 0 # number of time masking\nfreq_mask_num = 0 # number of frequency masking","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:41.170641Z","iopub.execute_input":"2022-11-23T07:09:41.17106Z","iopub.status.idle":"2022-11-23T07:09:41.187569Z","shell.execute_reply.started":"2022-11-23T07:09:41.171024Z","shell.execute_reply":"2022-11-23T07:09:41.186596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ZipDataset(torch.utils.data.Dataset):\n    \"\"\"\n    img, y = dataset[i]さく\n      img (np.float32): 2 x 360 x 180\n      y (np.float32): label 0 or 1\n    \"\"\"\n    def __init__(self, pos_len=6000, neg_len=7000, path='../input/g2net-generated-signals/archive',mod= 1000, noise = 0,tfms=False):\n        self.path = path\n        self.mod = int(mod)\n        self.noise = noise\n        self.noise_type = 2\n        self.tfms=tfms\n        self.pos_len = pos_len\n        self.neg_len = neg_len\n        self.len = pos_len + neg_len\n        \n        self.mixup = True\n        self.mixup_prob = 0.1\n        self.perm_pos = np.random.permutation(np.arange(self.pos_len))\n        self.perm_neg = np.random.permutation(np.arange(self.neg_len))\n    def __len__(self):\n        return self.len\n    def gen_noise(self, shape):\n        ns = 0.15\n        nr = 0.05\n\n        noise_shape = (360*4140)\n        noise_L_r = self.gen_noise(noise_shape)\n        noise_H_r = noise_L_r*(1-ns)+self.gen_noise(noise_shape)*ns\n        noise_L_i = noise_L_r*(1-nr)+self.gen_noise(noise_shape)*nr\n        noise_H_i = noise_H_r*(1-nr)+self.gen_noise(noise_shape)*nr\n\n        noise_r = np.stack([noise_L_r,noise_H_r]) *1e22\n        noise_i = np.stack([noise_L_i,noise_H_i]) *1e22\n        img_n = noise_r**2 + noise_i**2\n        return img_n\n    def get_negative(self,i):\n        file_name = f'../input/g2net-negative-gen/0_data_{self.mod*(1+(i)//self.mod)}/signals_{i%self.mod}.npy'\n        img = np.load(file_name).astype(np.float64)\n        for ch in range(2):\n            p = img[ch]\n            p /= np.mean(p)\n            img[ch] = p\n#         img = (img-np.min(img))/(np.max(img)-np.min(img))\n        y=0.0\n        return img, y\n    def get_positive(self, i):\n        file_name = f'../input/g2net-positive-gen/1_data_{self.mod*(1+(i)//self.mod)}/signals_{i%self.mod}.npy'\n        img = np.load(file_name).astype(np.float64)\n        for ch in range(2):\n            p = img[ch]\n            p /= np.mean(p)\n#             p = (p-np.min(p))/(np.max(p)-np.min(p))\n    #                 p = (p-np.min(p))/(np.max(p)-np.min(p))\n    #                 p = ndimage.median_filter(p, 2)\n    #                 p = ndimage.gaussian_filter(p, 1)\n\n    #                 blurred_f = ndimage.gaussian_filter(p, 3)\n    #                 filter_blurred_f = ndimage.gaussian_filter(blurred_f, 1)\n    #                 alpha = 30\n    #                 p = blurred_f + alpha * (blurred_f - filter_blurred_f)\n    #                 sx = ndimage.sobel(p, axis=0, mode='constant')\n    #                 sy = ndimage.sobel(p, axis=1, mode='constant')\n    #                 p = np.hypot(sx, sy)\n#             p = build_hough_space_fom_image(p)\n#             p = laplace_sharpen(p,1)\n            img[ch] = p\n\n#         img = ndimage.median_filter(img, 2)\n#         img = ndimage.gaussian_filter(img, 1)\n#         sx = ndimage.sobel(img, axis=0, mode='constant')\n#         sy = ndimage.sobel(img, axis=1, mode='constant')\n#         img = np.hypot(sx, sy)\n#         noise_id = int(random.random()*self.neg_len)\n#         noise_r = random.random()*0.05+0.95\n#         img = (np.sqrt(img)*(1-noise_r)+np.sqrt(self.get_negative(noise_id)[0])*noise_r)**2\n        y=1.0\n        return img, y\n    def get_noise(self):\n        return self.noise\n    def get_mixup(self, i, t):\n        if t==1:\n            mix_img = (self.get_positive(i)[0] + self.get_positive(self.perm_pos[i])[0])/2\n            if random.random() < 1/self.pos_len:\n                self.pos_perm = np.random.permutation(np.arange(self.pos_len))\n        else:\n            mix_img = (self.get_negative(i)[0] + self.get_negative(self.perm_neg[i])[0])/2\n            if random.random() < 1/self.neg_len:\n                self.neg_perm = np.random.permutation(np.arange(self.neg_len))\n        return mix_img, t\n    def __getitem__(self, i):\n        if i<self.pos_len:\n            if self.mixup and random.random() < self.mixup_prob:\n                img, y = self.get_mixup(i,1)\n            else:\n                img, y = self.get_positive(i)\n        else:\n            i = i-self.pos_len\n            if self.mixup and random.random() < self.mixup_prob:\n                img, y = self.get_mixup(i,0)\n            else:\n                img, y = self.get_negative(i)\n        if self.tfms:\n            if np.random.rand() <= flip_rate: # horizontal flip\n                img = np.flip(img, axis=1).copy()\n            if np.random.rand() <= flip_rate: # vertical flip\n                img = np.flip(img, axis=2).copy()\n            if np.random.rand() <= fre_shift_rate: # vertical shift\n                img = np.roll(img, np.random.randint(low=0, high=img.shape[1]), axis=1)\n            img = torch.from_numpy(img)\n\n            for _ in range(time_mask_num): # tima masking\n                img = transforms_time_mask(img)\n            for _ in range(freq_mask_num): # frequency masking\n                img = transforms_freq_mask(img)\n            img = img.numpy().astype(np.float32)\n#             img = ((img)/img.mean() ).astype(np.float32)\n        else:\n            img = img.astype(np.float32)\n#             img = ((img)/img.mean() ).astype(np.float32)\n            pass\n        return img, y\n    \nclass Dataset(torch.utils.data.Dataset):\n    \"\"\"\n    dataset = Dataset(data_type, df)\n\n    img, y = dataset[i]\n      img (np.float32): 2 x 360 x 128\n      y (np.float32): label 0 or 1\n    \"\"\"\n    def __init__(self, data_type, df,tfms=False):\n        self.data_type = data_type\n        self.df = df\n        self.tfms=tfms\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        \"\"\"\n        i (int): get ith data\n        \"\"\"\n        r = self.df.iloc[i]\n        y = np.float32(r.target)\n        file_id = r.id\n\n        img = np.empty((2, SHAPE_2[0], SHAPE_1[0]), dtype=np.float32)\n\n        filename = '%s/%s/%s.hdf5' % (di, self.data_type, file_id)\n        with h5py.File(filename, 'r') as f:\n            g = f[file_id]\n\n            for ch, s in enumerate(['H1', 'L1']):\n                p = normalize(np.asarray(g[s][\"SFTs\"][:, :SHAPE_1[0]*SHAPE_1[1]], np.complex128))\n#                 a = g[s]['SFTs'][:, :SHAPE_1[0]*SHAPE_1[1]].astype(np.complex128) * 1e22  # Fourier coefficient complex64\n                \n                \n#                 p /= np.mean(p)\n                \n                p = np.mean(p.reshape(360, SHAPE_1[0],SHAPE_1[1]), axis=2)\n                p = np.mean(p.reshape(SHAPE_2[0],SHAPE_2[1],SHAPE_1[0]), axis=1)\n                p = p.real**2 + p.imag**2  # power\n#                 p = (p-np.min(p))/(np.max(p)-np.min(p))\n#                 p = ndimage.median_filter(p, 2)\n#                 p = ndimage.gaussian_filter(p, 1)\n                \n#                 blurred_f = ndimage.gaussian_filter(p, 3)\n#                 filter_blurred_f = ndimage.gaussian_filter(blurred_f, 1)\n#                 alpha = 30\n#                 p = blurred_f + alpha * (blurred_f - filter_blurred_f)\n#                 sx = ndimage.sobel(p, axis=0, mode='constant')\n#                 sy = ndimage.sobel(p, axis=1, mode='constant')\n#                 p = np.hypot(sx, sy)\n#                 p = build_hough_space_fom_image(p)\n\n                img[ch] = p\n        if self.tfms:\n            if np.random.rand() <= flip_rate: # horizontal flip\n                img = np.flip(img, axis=1).copy()\n            if np.random.rand() <= flip_rate: # vertical flip\n                img = np.flip(img, axis=2).copy()\n            if np.random.rand() <= fre_shift_rate: # vertical shift\n                img = np.roll(img, np.random.randint(low=0, high=img.shape[1]), axis=1)\n            img = torch.from_numpy(img)\n\n            for _ in range(time_mask_num): # tima masking\n                img = transforms_time_mask(img)\n            for _ in range(freq_mask_num): # frequency masking\n                img = transforms_freq_mask(img)\n            img = img.numpy().astype(np.float32)\n#             img = ((img)/img.mean() ).astype(np.float32)\n            \n        else:\n            img = img.astype(np.float32)\n#             img = ((img)/img.mean() ).astype(np.float32)\n            pass\n        return img, y","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:41.191131Z","iopub.execute_input":"2022-11-23T07:09:41.19169Z","iopub.status.idle":"2022-11-23T07:09:41.227146Z","shell.execute_reply.started":"2022-11-23T07:09:41.191661Z","shell.execute_reply":"2022-11-23T07:09:41.226063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ValDataset(torch.utils.data.Dataset):\n    \"\"\"\n    img, y = dataset[i]\n      img (np.float32): 2 x 360 x 180\n      y (np.float32): label 0 or 1\n    \"\"\"\n    def __init__(self, pos_len=3000, neg_len=3000, path='../input/g2netgen',mod=1000, noise = 0):\n        self.path = path\n        self.mod = int(mod)\n        self.noise = noise\n        self.noise_type = 2\n        self.pos_len = pos_len\n        self.neg_len = neg_len\n        self.len = pos_len + neg_len\n        \n        self.mixup = True\n        self.mixup_prob = 0.1\n        self.perm_pos = np.random.permutation(np.arange(self.pos_len))\n        self.perm_neg = np.random.permutation(np.arange(self.neg_len))\n    def __len__(self):\n        return self.len\n    def gen_noise(self, shape):\n        ns = 0.15\n        nr = 0.05\n\n        noise_shape = (360*4140)\n        noise_L_r = self.gen_noise(noise_shape)\n        noise_H_r = noise_L_r*(1-ns)+self.gen_noise(noise_shape)*ns\n        noise_L_i = noise_L_r*(1-nr)+self.gen_noise(noise_shape)*nr\n        noise_H_i = noise_H_r*(1-nr)+self.gen_noise(noise_shape)*nr\n\n        noise_r = np.stack([noise_L_r,noise_H_r]) *1e22\n        noise_i = np.stack([noise_L_i,noise_H_i]) *1e22\n        img_n = noise_r**2 + noise_i**2\n        return img_n\n    def get_negative(self,i):\n        file_name = f'../input/g2net-negative-gen/0_data_{self.mod*(1+(i)//self.mod)}/signals_{i%self.mod}.npy'\n        img = np.load(file_name).astype(np.float64)\n        y=0.0\n        return img, y\n    def get_positive(self, i):\n        file_name = f'../input/g2net-positive-gen/1_data_{self.mod*(1+(i)//self.mod)}/signals_{i%self.mod}.npy'\n        img = np.load(file_name).astype(np.float64)\n        \n#         noise_id = int(random.random()*self.neg_len)\n#         noise_r = random.random()*0.05+0.95\n#         img = (np.sqrt(img)*(1-noise_r)+np.sqrt(self.get_negative(noise_id)[0])*noise_r)**2\n        y=1.0\n        return img, y\n    def get_noise(self):\n        return self.noise\n    def get_mixup(self, i, t):\n        if t==1:\n            mix_img = (self.get_positive(i)[0] + self.get_positive(self.perm_pos[i])[0])/2\n            if random.random() < 1/self.pos_len:\n                self.pos_perm = np.random.permutation(np.arange(self.pos_len))\n        else:\n            mix_img = (self.get_negative(i)[0] + self.get_negative(self.perm_neg[i])[0])/2\n            if random.random() < 1/self.neg_len:\n                self.neg_perm = np.random.permutation(np.arange(self.neg_len))\n        return mix_img, t\n    def __getitem__(self, i):\n        if i<self.pos_len:\n            if self.mixup and random.random() < self.mixup_prob:\n                img, y = self.get_mixup(i,1)\n            else:\n                img, y = self.get_positive(i)\n        else:\n            i = i-self.pos_len\n            if self.mixup and random.random() < self.mixup_prob:\n                img, y = self.get_mixup(i,0)\n            else:\n                img, y = self.get_negative(i)\n        img = ((img)/img.mean() ).astype(np.float32)\n        return img, y","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:41.228761Z","iopub.execute_input":"2022-11-23T07:09:41.229171Z","iopub.status.idle":"2022-11-23T07:09:41.248986Z","shell.execute_reply.started":"2022-11-23T07:09:41.229136Z","shell.execute_reply":"2022-11-23T07:09:41.247929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Data","metadata":{"_kg_hide-input":false}},{"cell_type":"code","source":"flip_rate = 0.6 # probability of applying the horizontal flip and vertical flip \nfre_shift_rate = 0.9 # probability of applying the vertical shift\ntime_mask_num = 1 # number of time masking\nfreq_mask_num = 2 # number of frequency masking","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:41.25089Z","iopub.execute_input":"2022-11-23T07:09:41.251292Z","iopub.status.idle":"2022-11-23T07:09:41.264845Z","shell.execute_reply.started":"2022-11-23T07:09:41.251256Z","shell.execute_reply":"2022-11-23T07:09:41.263281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = Dataset('train', df,tfms=False)\nimg_real, y = dataset[0]\ndataset = Dataset('test', submit)\nimg_test, y = dataset[18]\ndataset = ZipDataset(tfms=False)\nimg_label1_1, y = dataset[4]","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2022-11-23T07:09:41.267108Z","iopub.execute_input":"2022-11-23T07:09:41.267589Z","iopub.status.idle":"2022-11-23T07:09:42.721393Z","shell.execute_reply.started":"2022-11-23T07:09:41.267553Z","shell.execute_reply":"2022-11-23T07:09:42.720327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_real.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:42.723088Z","iopub.execute_input":"2022-11-23T07:09:42.72365Z","iopub.status.idle":"2022-11-23T07:09:42.730599Z","shell.execute_reply.started":"2022-11-23T07:09:42.723592Z","shell.execute_reply":"2022-11-23T07:09:42.729657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,3, figsize=(10, 10))\nplt.title('real image')\nax[0].set(title=f\"real\")\nax[1].set(title=f\"generated\")\nax[2].set(title=f\"test\")\nc0 = ax[0].imshow(img_real[0])\nc1 = ax[1].imshow(img_label1_1[0])\nc2 = ax[2].imshow(img_test[0])\nplt.savefig(\"train3.png\", dpi=300,format=\"png\")\nfig.colorbar(c0, ax=ax[0])\nfig.colorbar(c0, ax=ax[1])\nfig.colorbar(c0, ax=ax[2])\nplt.show()\n\nplt.title('value distribution')\n# plt.hist(img_test.flatten(),bins=100,alpha=0.1,color='b')\n# plt.hist(img_label1_1.flatten(),bins=100,alpha=0.1,color='r')\nplt.hist(img_real.flatten(),bins=100,alpha=0.1,color='g')\nplt.savefig(\"train4.png\", dpi=300,format=\"png\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:42.735421Z","iopub.execute_input":"2022-11-23T07:09:42.736101Z","iopub.status.idle":"2022-11-23T07:09:45.047756Z","shell.execute_reply.started":"2022-11-23T07:09:42.736063Z","shell.execute_reply":"2022-11-23T07:09:45.046766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset =  Dataset('test', submit)\nimg, y = dataset[2]\n\nplt.figure(figsize=(8, 3))\nplt.title('Spectrogram')\nplt.xlabel('time')\nplt.ylabel('frequency')\nplt.imshow(img[0, 300:360])  # zooming in for dataset[10]\nplt.colorbar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:45.051512Z","iopub.execute_input":"2022-11-23T07:09:45.051846Z","iopub.status.idle":"2022-11-23T07:09:45.916149Z","shell.execute_reply.started":"2022-11-23T07:09:45.051816Z","shell.execute_reply":"2022-11-23T07:09:45.914951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = Dataset('train', df)\nimg, y = dataset[10]\n    \nplt.figure(figsize=(8, 3))\nplt.title('Spectrogram')\nplt.xlabel('time')\nplt.ylabel('frequency')\nplt.imshow(img[0, 300:360])  # zooming in for dataset[10]\nplt.colorbar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:45.917568Z","iopub.execute_input":"2022-11-23T07:09:45.919486Z","iopub.status.idle":"2022-11-23T07:09:46.710181Z","shell.execute_reply.started":"2022-11-23T07:09:45.919431Z","shell.execute_reply":"2022-11-23T07:09:46.709218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class LargeKernel_debias(nn.Conv2d):\n    def forward(self, input: torch.Tensor):\n        finput = input.flatten(0, 1)[:, None]\n        target = abs(self.weight)\n        target = target / target.sum((-1, -2), True)\n        joined_kernel = torch.cat([self.weight, target], 0)\n        reals = target.new_zeros(\n            [1, 1] + [s + p * 2 for p, s in zip(self.padding, input.shape[-2:])]\n        )\n        reals[\n            [slice(None)] * 2 + [slice(p, -p) if p != 0 else slice(None) for p in self.padding]\n        ].fill_(1)\n        output, power = torch.nn.functional.conv2d(\n            finput, joined_kernel, padding=self.padding\n        ).chunk(2, 1)\n        ratio = torch.div(*torch.nn.functional.conv2d(reals, joined_kernel).chunk(2, 1))\n        output.sub_(power.mul_(ratio))\n        return output.unflatten(0, input.shape[:2]).flatten(1, 2)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:46.711542Z","iopub.execute_input":"2022-11-23T07:09:46.712545Z","iopub.status.idle":"2022-11-23T07:09:46.910744Z","shell.execute_reply.started":"2022-11-23T07:09:46.712508Z","shell.execute_reply":"2022-11-23T07:09:46.909492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, name, *, pretrained=False):\n        \"\"\"\n        name (str): timm model name, e.g. tf_efficientnet_b2_ns\n        \"\"\"\n        super().__init__()\n\n        # Use timm\n        model = timm.create_model(name, pretrained=pretrained, in_chans=2,num_classes = 2).to(device)\n        self.use_head=True\n        \n        if name[:9] == 'eca_nfnet':\n            clsf = 'head'\n            n_features = model._modules['head'].fc.in_features\n            model._modules[clsf].fc = nn.Identity()\n        elif name[:15] == 'tf_efficientnet':\n            clsf = model.default_cfg['classifier']\n            n_features = model._modules[clsf].in_features\n            model._modules[clsf] = nn.Identity()\n        else:\n            self.use_head=False\n            #placeholder\n            n_features=1\n        self.fc = nn.Sequential(\n#             nn.Conv2d(in_channels=2,out_channels=16,kernel_size=16),\n#             nn.ReLU(),\n#             nn.Conv2d(in_channels=16,out_channels=16,kernel_size=16),\n#             nn.ReLU(),\n#             nn.MaxPool2d(kernel_size=4),\n#             nn.Conv2d(in_channels=16,out_channels=32,kernel_size=8),\n#             nn.ReLU(),\n#             nn.Conv2d(in_channels=32,out_channels=32,kernel_size=8),\n#             nn.ReLU(),\n#             nn.MaxPool2d(kernel_size=4),\n#             nn.Conv2d(in_channels=32,out_channels=64,kernel_size=4),\n#             nn.ReLU(),\n#             nn.Conv2d(in_channels=64,out_channels=64,kernel_size=4),\n#             nn.ReLU(),\n#             nn.MaxPool2d(kernel_size=4),\n#             nn.Linear(in_features=5, out_features=16),\n#             nn.ReLU(),\n#             nn.Linear(in_features=16, out_features=16),\n#             nn.ReLU(),\n#             nn.Linear(in_features=16, out_features=16),\n#             nn.ReLU(),\n#             nn.Linear(in_features=16, out_features=2),\n#             nn.Linear(in_features=n_features, out_features=64),\n#             nn.ReLU(),\n#             nn.Linear(in_features=64, out_features=64),\n#             nn.ReLU(),\n#             nn.Linear(in_features=64, out_features=64),\n#             nn.ReLU(),\n            nn.Linear(in_features=n_features, out_features=1),\n#             nn.Softmax(dim=1)\n        )\n        self.model = model\n\n    def forward(self, x):\n        if IMG_SIZE:\n            x = F.interpolate(x,IMG_SIZE)\n        x = self.model(x)\n#         print(x.shape)\n        if self.use_head:\n            x = self.fc(x)\n#             print(x.shape)\n        else:\n            x = x[:,0]\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:25:20.88838Z","iopub.execute_input":"2022-11-23T07:25:20.889063Z","iopub.status.idle":"2022-11-23T07:25:20.901494Z","shell.execute_reply.started":"2022-11-23T07:25:20.889015Z","shell.execute_reply":"2022-11-23T07:25:20.900237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict and evaluate","metadata":{}},{"cell_type":"code","source":"def evaluate(model, loader_val, *, compute_score=True, pbar=None):\n    \"\"\"\n    Predict and compute loss and score\n    \"\"\"\n    tb = time.time()\n    was_training = model.training\n    model.eval()\n\n    loss_sum = 0.0\n    n_sum = 0\n    y_all = []\n    y_pred_all = []\n\n    if pbar is not None:\n        pbar = tqdm(desc='Predict', nrows=78, total=pbar)\n\n    for img, y in loader_val:\n        n = y.size(0)\n        img = img.to(device)\n        y = y.to(device)\n\n        with torch.no_grad():\n            y_pred = model(img)\n        loss = criterion(y_pred.view(-1), y)\n\n        n_sum += n\n        loss_sum += n * loss.item()\n\n        y_all.append(y.cpu().detach().numpy())\n        y_pred_all.append(y_pred.sigmoid().squeeze().cpu().detach().numpy())\n\n        if pbar is not None:\n            pbar.update(len(img))\n        \n        del loss, y_pred, img, y\n        gc.collect()\n\n    loss_val = loss_sum / n_sum\n\n    y = np.concatenate(y_all)\n    y_pred = np.concatenate(y_pred_all)\n\n    score = roc_auc_score(y, y_pred) if compute_score else None\n\n    ret = {'loss': loss_val,\n           'score': score,\n           'y': y,\n           'y_pred': y_pred,\n           'time': time.time() - tb}\n    \n    model.train(was_training)  # back to train from eval if necessary\n    gc.collect()\n    return ret","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:21:08.17351Z","iopub.execute_input":"2022-11-23T07:21:08.1739Z","iopub.status.idle":"2022-11-23T07:21:08.186127Z","shell.execute_reply.started":"2022-11-23T07:21:08.173867Z","shell.execute_reply":"2022-11-23T07:21:08.184848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pretrain","metadata":{}},{"cell_type":"code","source":"# epochs = 5\n# batch_size = 32\n# weight_decay = 1e-6\n# max_grad_norm = 1000\n\n# lr_max = 4e-4\n# epochs_warmup = 1.0\n\n# torch.manual_seed(42)\n\n# # Train - val split\n# dataset_train = ZipDataset(tfms=True)\n# dataset_val = Dataset('train', df,tfms=True)\n\n# loader_train = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size,\n#                  num_workers=0, pin_memory=False, shuffle=True, drop_last=True)\n# loader_val = torch.utils.data.DataLoader(dataset_val, batch_size=batch_size,\n#                  num_workers=0, pin_memory=False)\n\n# # Model and optimizer\n# model = Model(model_name, pretrained=True)\n# model.to(device)\n# model = DataParallel(model, device_ids=gpus, output_device=gpus[0])\n# model.train()\n\n# optimizer = torch.optim.Adam(model.parameters(), lr=lr_max, weight_decay=weight_decay)\n\n# # Learning-rate schedule\n# nbatch = len(loader_train)\n# warmup = epochs_warmup * nbatch  # number of warmup steps\n# nsteps = epochs * nbatch        # number of total steps\n\n# scheduler = CosineLRScheduler(optimizer,\n#               warmup_t=warmup, warmup_lr_init=0.0, warmup_prefix=True, # 1 epoch of warmup\n#               t_initial=(nsteps - warmup), lr_min=1e-6)                # 3 epochs of cosine\n\n# time_val = 0.0\n# lrs = []\n# best_loss = 1e10\n# base = 1e-3\n# tb = time.time()\n# print('Epoch   loss          score   lr')\n# for iepoch in range(epochs):\n#     loss_sum = 0.0\n#     n_sum = 0\n\n#     # Train\n#     for ibatch, (img, y) in enumerate(loader_train):\n#         n = y.size(0)\n#         img = img.to(device)\n#         y = y.to(device)\n\n#         optimizer.zero_grad()\n\n#         y_pred = model(img)\n#         loss = criterion(y_pred.view(-1), y)\n\n#         loss_train = loss.item()\n#         loss_sum += n * loss_train\n#         n_sum += n\n\n#         loss.backward()\n\n#         grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),\n#                                                    max_grad_norm)\n#         optimizer.step()\n\n#         scheduler.step(iepoch * nbatch + ibatch + 1)\n#         lrs.append(optimizer.param_groups[0]['lr'])            \n\n#     # Evaluate\n#     val = evaluate(model, loader_val)\n#     time_val += val['time']\n#     loss_train = loss_sum / n_sum\n#     lr_now = optimizer.param_groups[0]['lr']\n#     dt = (time.time() - tb) / 60\n#     print('Epoch %d %.4f %.4f %.4f  %.2e  %.2f min' %\n#           (iepoch + 1, loss_train, val['loss'], val['score'], lr_now, dt))\n#     if val['loss'] < base:\n#         pass\n#     elif val['loss'] < best_loss:\n#         best_loss = val['loss']\n#         # Save model\n#         ofilename = 'model_pretrain.pytorch'\n#         torch.save(model.state_dict(), ofilename)\n#         print(ofilename, 'written')\n#     del val\n#     gc.collect()\n\n# dt = time.time() - tb\n# print('Training done %.2f min total, %.2f min val' % (dt / 60, time_val / 60))","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:09:46.973276Z","iopub.execute_input":"2022-11-23T07:09:46.973705Z","iopub.status.idle":"2022-11-23T07:09:46.990819Z","shell.execute_reply.started":"2022-11-23T07:09:46.973668Z","shell.execute_reply":"2022-11-23T07:09:46.988531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"nfold = 5\nkfold = StratifiedKFold(n_splits=nfold, random_state=42, shuffle=True)\n\nepochs = 25\nbatch_size = 16\nnum_workers = 2\nweight_decay = 1e-6\nmax_grad_norm = 1000\n\nlr_max = 4e-4\nepochs_warmup = 1.0 \n\nflip_rate = 0.5 # probability of applying the horizontal flip and vertical flip \nfre_shift_rate = 0.9 # probability of applying the vertical shift\ntime_mask_num = 1 # number of time masking\nfreq_mask_num = 2 # number of frequency masking\n\nfor ifold, (idx_train, idx_test) in enumerate(kfold.split(dataset, df['target'])):\n    print('Fold %d/%d' % (ifold, nfold))\n    torch.manual_seed(42 + ifold + 1)\n\n    # Train - val split\n    dataset_train = Dataset('train', df.iloc[idx_train],tfms=True)\n    dataset_val = Dataset('train', df.iloc[idx_test])\n\n    loader_train = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size,\n                     num_workers=num_workers, pin_memory=True, shuffle=True, drop_last=True)\n    loader_val = torch.utils.data.DataLoader(dataset_val, batch_size=batch_size,\n                     num_workers=num_workers, pin_memory=True)\n\n    # Model and optimizer\n    model = Model(model_name, pretrained=PRETRAIN)\n#     if PRETRAIN:\n#         model.load_state_dict(torch.load('/kaggle/input/g2netdetectingcontinuousgravitationalwavesv0/group0/model_best.pth'),False)\n    model.to(device)\n    model = DataParallel(model, device_ids=gpus, output_device=gpus[0])\n    model.train()\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr_max, weight_decay=weight_decay)\n\n    # Learning-rate schedule\n    nbatch = len(loader_train)\n    warmup = epochs_warmup * nbatch  # number of warmup steps\n    nsteps = epochs * nbatch        # number of total steps\n\n    scheduler = CosineLRScheduler(optimizer,\n                  warmup_t=warmup, warmup_lr_init=0.0, warmup_prefix=True, # 1 epoch of warmup\n                  t_initial=(nsteps - warmup), lr_min=1e-6)                # 3 epochs of cosine\n    \n    time_val = 0.0\n    lrs = []\n    best_val_loss = 1e10\n    best_val_score=0\n    tb = time.time()\n    print('Epoch   loss          score   lr')\n    for iepoch in range(epochs):\n        loss_sum = 0.0\n        n_sum = 0\n\n        # Train\n        for ibatch, (img, y) in enumerate(loader_train):\n            n = y.size(0)\n            img = img.to(device)\n            y = y.to(device)\n\n            optimizer.zero_grad()\n\n            y_pred = model(img)\n            loss = criterion(y_pred.view(-1), y)\n\n            loss_train = loss.item()\n            loss_sum += n * loss_train\n            n_sum += n\n\n            loss.backward()\n\n            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),\n                                                       max_grad_norm)\n            optimizer.step()\n            \n            scheduler.step(iepoch * nbatch + ibatch + 1)\n            lrs.append(optimizer.param_groups[0]['lr'])            \n\n        # Evaluate\n        val = evaluate(model, loader_val)\n        time_val += val['time']\n        loss_train = loss_sum / n_sum\n        lr_now = optimizer.param_groups[0]['lr']\n        dt = (time.time() - tb) / 60\n        print('Epoch %d %.4f %.4f %.4f  %.2e  %.2f min' %\n              (iepoch + 1, loss_train, val['loss'], val['score'], lr_now, dt))\n        if val['loss']<best_val_loss:\n            best_val_loss = val['loss']\n            ofilename = 'model%d.pytorch' % ifold\n            torch.save(model.state_dict(), ofilename)\n            print(ofilename, 'written')\n    dt = time.time() - tb\n    print('Training done %.2f min total, %.2f min val' % (dt / 60, time_val / 60))\nprint('AVG LOSS:', np.mean(np.array(best_val_loss)))\nprint('AVG SCORE:', np.mean(np.array(best_val_score)))","metadata":{"execution":{"iopub.status.busy":"2022-11-23T07:25:25.633659Z","iopub.execute_input":"2022-11-23T07:25:25.634135Z","iopub.status.idle":"2022-11-23T07:28:23.982324Z","shell.execute_reply.started":"2022-11-23T07:25:25.634099Z","shell.execute_reply":"2022-11-23T07:28:23.973041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict","metadata":{}},{"cell_type":"code","source":"preds=[]","metadata":{"execution":{"iopub.status.busy":"2022-11-07T04:05:22.398085Z","iopub.status.idle":"2022-11-07T04:05:22.398835Z","shell.execute_reply.started":"2022-11-07T04:05:22.398543Z","shell.execute_reply":"2022-11-07T04:05:22.398583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load model (if necessary)\n# pred = []\n# filename = 'model_pretrain.pytorch'\n# model = Model(model_name, pretrained=False)\n# model.to(device)\n# model = DataParallel(model, device_ids=gpus, output_device=gpus[0])\n# model.load_state_dict(torch.load(filename, map_location=device),False)\n# model.eval()\n\n# # Predict\n# submit = pd.read_csv(di + '/sample_submission.csv')\n# dataset_test = Dataset('test', submit)\n# loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=32,\n#                                           num_workers=2, pin_memory=True)\n\n# test = evaluate(model, loader_test, compute_score=False, pbar=len(submit))\n# pred.append(list(test['y_pred']))\n# gc.collect()\n# print(f'{filename} predicted')\n# # Write prediction\n# p = pd.DataFrame(pred).T\n# p.head()\n# submit['target'] = pd.DataFrame(p.mean(axis=1))\n# submit.to_csv('submission_pre.csv', index=False)\n# print('target range [%.2f, %.2f]' % (submit['target'].min(), submit['target'].max()))\n# del pred,model,test","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load model (if necessary)\n\nfilenames = [f'model{x}.pytorch' for x in range(5)]\nfor filename in tqdm(filenames):\n    model = Model(model_name, pretrained=False)\n    model.to(device)\n    model = DataParallel(model, device_ids=gpus, output_device=gpus[0])\n    model.load_state_dict(torch.load(filename, map_location=device),False)\n    model.eval()\n\n    # Predict\n    submit = pd.read_csv(di + '/sample_submission.csv')\n    dataset_test = Dataset('test', submit)\n    loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=32,\n                                              num_workers=2, pin_memory=True)\n\n    test = evaluate(model, loader_test, compute_score=False, pbar=len(submit))\n    preds.append(list(test['y_pred']))\n    gc.collect()\n    print(f'{filename} predicted')\n# Write prediction\n","metadata":{"execution":{"iopub.status.busy":"2022-11-07T04:05:22.40018Z","iopub.status.idle":"2022-11-07T04:05:22.40094Z","shell.execute_reply.started":"2022-11-07T04:05:22.400662Z","shell.execute_reply":"2022-11-07T04:05:22.400697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = pd.DataFrame(preds).T\nx.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-07T04:05:22.40223Z","iopub.status.idle":"2022-11-07T04:05:22.402979Z","shell.execute_reply.started":"2022-11-07T04:05:22.402705Z","shell.execute_reply":"2022-11-07T04:05:22.402746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit['target'] = pd.DataFrame(x.mean(axis=1))\nsubmit.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-07T04:05:22.404413Z","iopub.status.idle":"2022-11-07T04:05:22.405133Z","shell.execute_reply.started":"2022-11-07T04:05:22.404878Z","shell.execute_reply":"2022-11-07T04:05:22.404903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('target range [%.2f, %.2f]' % (submit['target'].min(), submit['target'].max()))","metadata":{},"execution_count":null,"outputs":[]}]}