{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Basic spectrogram image classification\n\nPower spectrum is averaged over every N = 32 timesteps:\n\n$$ P(\\omega) = \\frac{1}{N} \\sum_n^N \\left|a(\\omega, t_n) \\right|^2 $$\n\nImage size becomes: 2 channels x 360 frequencies x 128 timesteps.\n\nThe run time is dominated by data loading but I am not optimizing for that.\n\nVersion 4: Fix output target value to [0, 1] with `y_pred.sigmoid()`.","metadata":{}},{"cell_type":"code","source":"# Use timm pretrained image model\n! pip3 install timm","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:05:55.182872Z","iopub.execute_input":"2022-11-19T00:05:55.183391Z","iopub.status.idle":"2022-11-19T00:06:08.106933Z","shell.execute_reply.started":"2022-11-19T00:05:55.183346Z","shell.execute_reply":"2022-11-19T00:06:08.105611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport time\nimport h5py\nimport timm\nimport torch\nimport torch.nn as nn\nimport cv2\n\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nfrom timm.scheduler import CosineLRScheduler\n\ndevice = torch.device('cuda')\ncriterion = nn.BCEWithLogitsLoss()\n\n# Train metadata\ndi = '/kaggle/input/g2net-detecting-continuous-gravitational-waves'\ndf = pd.read_csv(di + '/train_labels.csv')\ndf = df[df.target >= 0]  # Remove 3 unknowns (target = -1)","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:06:08.110834Z","iopub.execute_input":"2022-11-19T00:06:08.111201Z","iopub.status.idle":"2022-11-19T00:06:10.491527Z","shell.execute_reply.started":"2022-11-19T00:06:08.111168Z","shell.execute_reply":"2022-11-19T00:06:10.490463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    \"\"\"\n    dataset = Dataset(data_type, df)\n\n    img, y = dataset[i]\n      img (np.float32): 2 x 360 x 360\n      y (np.float32): label 0 or 1\n    \"\"\"\n    def __init__(self, data_type, df):\n        self.data_type = data_type\n        self.df = df\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        \"\"\"\n        i (int): get ith data\n        \"\"\"\n        r = self.df.iloc[i]\n        y = np.float32(r.target)\n        file_id = r.id\n\n        img = np.empty((2, 360, 360), dtype=np.float32)\n\n        filename = '%s/%s/%s.hdf5' % (di, self.data_type, file_id)\n        with h5py.File(filename, 'r') as f:\n            g = f[file_id]\n\n            for ch, s in enumerate(['H1', 'L1']):\n                a = g[s]['SFTs'][:, :4096] * 1e22  # Fourier coefficient complex64\n\n                p = a.real**2 + a.imag**2  # power\n                p /= np.mean(p)  # normalize\n                p = np.mean(p.reshape(360, 128, 32), axis=2)  # compress 4096 -> 360\n                resized_image = cv2.resize(p, (360, 360), interpolation = cv2.INTER_AREA)\n                img[ch] = resized_image\n\n        return img, y","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:06:10.493736Z","iopub.execute_input":"2022-11-19T00:06:10.494406Z","iopub.status.idle":"2022-11-19T00:06:10.505458Z","shell.execute_reply.started":"2022-11-19T00:06:10.494361Z","shell.execute_reply":"2022-11-19T00:06:10.504395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = Dataset('train', df)\nimg, y = dataset[10]\n\nplt.figure(figsize=(8, 3))\nplt.title('Spectrogram')\nplt.xlabel('time')\nplt.ylabel('frequency')\nplt.imshow(img[0, 300:360])  # zooming in for dataset[10]\nplt.colorbar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:06:10.510227Z","iopub.execute_input":"2022-11-19T00:06:10.510512Z","iopub.status.idle":"2022-11-19T00:06:11.506255Z","shell.execute_reply.started":"2022-11-19T00:06:10.510484Z","shell.execute_reply":"2022-11-19T00:06:11.504664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n        self.conv_layer1 = nn.Conv2d(in_channels=2, out_channels=32, kernel_size=3, stride=1)\n        self.conv_layer2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=4, stride=1)\n        self.conv_layer3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=2, stride=1)\n        self.conv_layer4 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=2, stride=1)\n        self.conv_layer5 = nn.Conv2d(in_channels=256, out_channels=2048, kernel_size=7, stride=1)\n        \n        self.pooling_layer1 = nn.MaxPool2d(kernel_size=2)\n        self.pooling_layer2 = nn.AvgPool2d(kernel_size=3)\n        self.pooling_layer3 = nn.AvgPool2d(kernel_size=4)\n        \n        self.linear_layer1 = nn.Linear(in_features=2048, out_features=1024)\n        self.linear_layer2 = nn.Linear(in_features=1024, out_features=512)\n        self.linear_layer3 = nn.Linear(in_features=512, out_features=1)\n        \n        self.relu = nn.ReLU()\n        \n    def forward(self, inputs):\n        output_1 = self.relu(self.conv_layer1(inputs))\n        output_2 = self.pooling_layer1(output_1)\n        output_3 = self.relu(self.conv_layer2(output_2))\n        output_4 = self.pooling_layer1(output_3)\n        output_5 = self.relu(self.conv_layer3(output_4))\n        output_6 = self.pooling_layer2(output_5)\n        output_7 = self.relu(self.conv_layer4(output_6))\n        output_8 = self.pooling_layer3(output_7)\n        output_9 = self.relu(self.conv_layer5(output_8))\n        output_10 = torch.flatten(output_9, 1)\n        \n        output_11 = self.relu(self.linear_layer1(output_10))\n        output_12 = self.relu(self.linear_layer2(output_11))\n        output = self.linear_layer3(output_12)\n        \n        return output","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:06:11.507468Z","iopub.execute_input":"2022-11-19T00:06:11.508023Z","iopub.status.idle":"2022-11-19T00:06:11.521067Z","shell.execute_reply.started":"2022-11-19T00:06:11.507977Z","shell.execute_reply":"2022-11-19T00:06:11.519793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model1(nn.Module):\n    def __init__(self):\n        super(Model1, self).__init__()\n        self.conv_layer1 = nn.Conv2d(in_channels=2, out_channels=128, kernel_size=3, stride=1, padding=1)\n        self.conv_layer2 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1)\n        self.conv_layer3 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, stride=1, padding=1)\n        self.conv_layer4 = nn.Conv2d(in_channels=256, out_channels=1024, kernel_size=6, stride=1)\n        self.conv_layer5 = nn.Conv2d(in_channels=1024, out_channels=2048, kernel_size=8, stride=1)\n        self.pooling_layer1 = nn.MaxPool2d(kernel_size=2)\n        self.pooling_layer2 = nn.MaxPool2d(kernel_size=2)\n        self.pooling_layer3 = nn.MaxPool2d(kernel_size=2)\n        self.pooling_layer4 = nn.MaxPool2d(kernel_size=5)\n        \n        self.linear_layer1 = nn.Linear(in_features=2048, out_features=1000)\n        self.linear_layer2 = nn.Linear(in_features=1000, out_features=1)\n        \n        self.relu = nn.ReLU()\n        \n    def forward(self, inputs):\n        output_1 = self.relu(self.conv_layer1(inputs))\n        output_2 = self.pooling_layer1(output_1)\n        output_3 = self.relu(self.conv_layer2(output_2))\n        output_4 = self.pooling_layer2(output_3)\n        output_5 = self.relu(self.conv_layer3(output_4))\n        output_6 = self.pooling_layer3(output_5)\n        output_7 = self.relu(self.conv_layer4(output_6))\n        output_8 = self.pooling_layer4(output_7)\n        output_9 = self.relu(self.conv_layer5(output_8))\n        output_10 = torch.flatten(output_9, 1)\n        \n        output_11 = self.relu(self.linear_layer1(output_10))\n        output = self.linear_layer2(output_11)\n        \n        return output","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:06:11.522877Z","iopub.execute_input":"2022-11-19T00:06:11.523622Z","iopub.status.idle":"2022-11-19T00:06:11.538493Z","shell.execute_reply.started":"2022-11-19T00:06:11.52358Z","shell.execute_reply":"2022-11-19T00:06:11.537655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, name, *, pretrained=False):\n        \"\"\"\n        name (str): timm model name, e.g. tf_efficientnet_b2_ns\n        \"\"\"\n        super().__init__()\n\n        # Use timm\n        model = timm.create_model(name, pretrained=pretrained, in_chans=2)\n\n        clsf = model.default_cfg['classifier']\n        n_features = model._modules[clsf].in_features\n        model._modules[clsf] = nn.Identity()\n\n        self.fc = nn.Linear(n_features, 1)\n        self.model = model\n\n    def forward(self, x):\n        x = self.model(x)\n        x = self.fc(x)\n        return x","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict and evaluate","metadata":{}},{"cell_type":"code","source":"def evaluate(model, loader_val, *, compute_score=True, pbar=None):\n    \"\"\"\n    Predict and compute loss and score\n    \"\"\"\n    tb = time.time()\n    was_training = model.training\n    model.eval()\n\n    loss_sum = 0.0\n    n_sum = 0\n    y_all = []\n    y_pred_all = []\n\n    if pbar is not None:\n        pbar = tqdm(desc='Predict', nrows=78, total=pbar)\n\n    for img, y in loader_val:\n        n = y.size(0)\n        img = img.to(device)\n        y = y.to(device)\n\n        with torch.no_grad():\n            y_pred = model(img.to(device))\n        loss = criterion(y_pred.view(-1), y)\n\n        n_sum += n\n        loss_sum += n * loss.item()\n\n        y_all.append(y.cpu().detach().numpy())\n        y_pred_all.append(y_pred.sigmoid().squeeze().cpu().detach().numpy())\n\n        if pbar is not None:\n            pbar.update(len(img))\n        \n        del loss, y_pred, img, y\n\n    loss_val = loss_sum / n_sum\n\n    y = np.concatenate(y_all)\n    y_pred = np.concatenate(y_pred_all)\n\n    score = roc_auc_score(y, y_pred) if compute_score else None\n\n    ret = {'loss': loss_val,\n           'score': score,\n           'y': y,\n           'y_pred': y_pred,\n           'time': time.time() - tb}\n    \n    model.train(was_training)  # back to train from eval if necessary\n\n    return ret","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:06:11.54026Z","iopub.execute_input":"2022-11-19T00:06:11.541013Z","iopub.status.idle":"2022-11-19T00:06:11.553444Z","shell.execute_reply.started":"2022-11-19T00:06:11.540974Z","shell.execute_reply":"2022-11-19T00:06:11.552564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"model = Model().to(device)\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:05:44.916592Z","iopub.status.idle":"2022-11-19T00:05:44.917835Z","shell.execute_reply.started":"2022-11-19T00:05:44.917529Z","shell.execute_reply":"2022-11-19T00:05:44.91756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1 = Model1().to(device)\nprint(model1)","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:06:11.555118Z","iopub.execute_input":"2022-11-19T00:06:11.555844Z","iopub.status.idle":"2022-11-19T00:06:16.439629Z","shell.execute_reply.started":"2022-11-19T00:06:11.555805Z","shell.execute_reply":"2022-11-19T00:06:16.438473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = 'tf_efficientnet_b2_ns'\nnfold = 5\nkfold = KFold(n_splits=nfold, random_state=42, shuffle=True)\n\nepochs = 4\nbatch_size = 16\nnum_workers = 2\nweight_decay = 1e-6\nmax_grad_norm = 1000\n\nlr_max = 4e-4\nepochs_warmup = 1.0\n\nfor ifold, (idx_train, idx_test) in enumerate(kfold.split(df)):\n    print('Fold %d/%d' % (ifold, nfold))\n    torch.manual_seed(42 + ifold + 1)\n\n    # Train - val split\n    dataset_train = Dataset('train', df.iloc[idx_train])\n    dataset_val = Dataset('train', df.iloc[idx_test])\n\n    loader_train = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size,\n                     num_workers=num_workers, pin_memory=True, shuffle=True, drop_last=True)\n    loader_val = torch.utils.data.DataLoader(dataset_val, batch_size=batch_size,\n                     num_workers=num_workers, pin_memory=True)\n\n    # Model and optimizer\n    model = Model1()\n    model.to(device)\n    model.train()\n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=lr_max, weight_decay=weight_decay)\n\n    # Learning-rate schedule\n    nbatch = len(loader_train)\n    warmup = epochs_warmup * nbatch  # number of warmup steps\n    nsteps = epochs * nbatch        # number of total steps\n\n    scheduler = CosineLRScheduler(optimizer,\n                  warmup_t=warmup, warmup_lr_init=0.0, warmup_prefix=True, # 1 epoch of warmup\n                  t_initial=(nsteps - warmup), lr_min=1e-6)                # 3 epochs of cosine\n    \n    time_val = 0.0\n    lrs = []\n\n    tb = time.time()\n    print('Epoch   loss          score   lr')\n    for iepoch in range(epochs):\n        loss_sum = 0.0\n        n_sum = 0\n\n        # Train\n        for ibatch, (img, y) in enumerate(loader_train):\n            n = y.size(0)\n            img = img.to(device)\n            y = y.to(device)\n\n            optimizer.zero_grad()\n\n            y_pred = model(img)\n            loss = criterion(y_pred.view(-1), y)\n\n            loss_train = loss.item()\n            loss_sum += n * loss_train\n            n_sum += n\n\n            loss.backward()\n\n            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),\n                                                       max_grad_norm)\n            optimizer.step()\n            \n            scheduler.step(iepoch * nbatch + ibatch + 1)\n            lrs.append(optimizer.param_groups[0]['lr'])            \n\n        # Evaluate\n        val = evaluate(model, loader_val)\n        time_val += val['time']\n        loss_train = loss_sum / n_sum\n        lr_now = optimizer.param_groups[0]['lr']\n        dt = (time.time() - tb) / 60\n        print('Epoch %d %.4f %.4f %.4f  %.2e  %.2f min' %\n              (iepoch + 1, loss_train, val['loss'], val['score'], lr_now, dt))\n\n    dt = time.time() - tb\n    print('Training done %.2f min total, %.2f min val' % (dt / 60, time_val / 60))\n\n    # Save model\n    ofilename = 'model%d.pytorch' % ifold\n    torch.save(model.state_dict(), ofilename)\n    print(ofilename, 'written')\n\n    break  # 1 fold only","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:06:47.996214Z","iopub.execute_input":"2022-11-19T00:06:47.99665Z","iopub.status.idle":"2022-11-19T00:13:59.956837Z","shell.execute_reply.started":"2022-11-19T00:06:47.996609Z","shell.execute_reply":"2022-11-19T00:13:59.954601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('LR Schedule: Cosine with linear warmup')\nplt.xlabel('steps')\nplt.ylabel('learning rate')\nplt.plot(lrs)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-19T00:13:59.971872Z","iopub.execute_input":"2022-11-19T00:13:59.97524Z","iopub.status.idle":"2022-11-19T00:14:00.406937Z","shell.execute_reply.started":"2022-11-19T00:13:59.975167Z","shell.execute_reply":"2022-11-19T00:14:00.405985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict and submit","metadata":{}},{"cell_type":"code","source":"# Load model (if necessary)\nmodel = Model1()\nfilename = 'model0.pytorch'\nmodel.to(device)\nmodel.load_state_dict(torch.load(filename, map_location=device))\nmodel.eval()\n\n# Predict\nsubmit = pd.read_csv(di + '/sample_submission.csv')\ndataset_test = Dataset('test', submit)\nloader_test = torch.utils.data.DataLoader(dataset_test, batch_size=16,\n                                          num_workers=num_workers, pin_memory=True)\n\ntest = evaluate(model, loader_test, compute_score=False, pbar=len(submit))\n\n# Write prediction\nsubmit['target'] = test['y_pred']\nsubmit.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-19T01:12:53.196944Z","iopub.execute_input":"2022-11-19T01:12:53.197334Z","iopub.status.idle":"2022-11-19T01:36:25.236733Z","shell.execute_reply.started":"2022-11-19T01:12:53.197291Z","shell.execute_reply":"2022-11-19T01:36:25.233862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('target range [%.2f, %.2f]' % (submit['target'].min(), submit['target'].max()))","metadata":{"execution":{"iopub.status.busy":"2022-11-19T01:39:43.006382Z","iopub.execute_input":"2022-11-19T01:39:43.008472Z","iopub.status.idle":"2022-11-19T01:39:43.02133Z","shell.execute_reply.started":"2022-11-19T01:39:43.008408Z","shell.execute_reply":"2022-11-19T01:39:43.018877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}