{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip install h5py","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install timm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport h5py\nimport timm\nimport matplotlib.pyplot as plt\nimport seaborn\nimport time\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader\nfrom torch.optim.lr_scheduler import StepLR\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom tqdm.auto import tqdm\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nfrom timm.scheduler import CosineLRScheduler\n\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndi = '/kaggle/input/g2net-detecting-continuous-gravitational-waves'\n\n%matplotlib inline","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/train_labels.csv')\nsubmission = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Removing the negative labels\ntrain_labels = train_labels[train_labels.target>=0]\ntrain_labels.target.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#defining a configuration\nclass CFG:\n    model_name = 'tf_efficientnet_b7_ns'\n    target_size = 1\n    nfold = 5\n    batch_size = 16\n    epochs = 16\n    num_workers = 2\n    lr = 1e-3\n    weight_decay = 1e-6\n    train = True\n    seed = 42\n    score_method = 'roc_auc_score'\n    scheduler_type = 'CosineLRScheduler'\n    optimizer_type = 'AdamW'\n    loss_type = 'BCEWithLogitsLoss'\n    max_grad_norm = 1000\n    lr_max = 4e-4\n    epochs_warmup = 1.0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_criterion():\n    if CFG.loss_type == 'CrossEntropyLoss':\n        return nn.CrossEntropyLoss()\n    if CFG.loss_type == 'BCEWithLogitsLoss':\n        return nn.BCEWithLogitsLoss()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_optimizer(model):\n    if CFG.optimizer_type == 'Adam':\n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\n    if CFG.optimizer_type == 'AdamW':\n        optimizer = torch.optim.AdamW(model.parameters(), lr=CFG.lr_max, weight_decay=CFG.weight_decay)\n    return optimizer","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_scheduler(optimizer, warmup, nsteps):\n    if CFG.scheduler_type == 'StepLR':\n        scheduler = StepLR(optimizer, step_size=2, gamma=0.1, verbose=True)\n    if CFG.scheduler_type == 'CosineLRScheduler':\n        scheduler = CosineLRScheduler(optimizer,\n                                      warmup_t=warmup, warmup_lr_init=0.0, warmup_prefix=True,\n                                      t_initial=(nsteps - warmup), lr_min=1e-6) \n    return scheduler","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_score(y_true, y_pred):\n    if CFG.score_method == \"roc_auc_score\":\n        score = roc_auc_score(y_true, y_pred)\n    if CFG.score_method == \"accuracy_score\":\n        score = accuracy_score(y_true, y_pred)\n    return score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    \"\"\"\n    dataset = Dataset(data_type, df)\n\n    img, y = dataset[i]\n      img (np.float32): 2 x 360 x 128\n      y (np.float32): label 0 or 1\n    \"\"\"\n    def __init__(self, data_type, df):\n        self.data_type = data_type\n        self.df = df\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        \"\"\"\n        i (int): get ith data\n        \"\"\"\n        r = self.df.iloc[i]\n        y = np.float32(r.target)\n        file_id = r.id\n\n        img = np.empty((2, 360, 128), dtype=np.float32)\n\n        filename = '%s/%s/%s.hdf5' % (di, self.data_type, file_id)\n        with h5py.File(filename, 'r') as f:\n            g = f[file_id]\n\n            for ch, s in enumerate(['H1', 'L1']):\n                a = g[s]['SFTs'][:, :4096] * 1e22  # Fourier coefficient complex64\n\n                p = a.real**2 + a.imag**2  # power\n                p /= np.mean(p)  # normalize\n                p = np.mean(p.reshape(360, 128, 32), axis=2)  # compress 4096 -> 128\n\n                img[ch] = p\n\n        return img, y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = Dataset('train', train_labels)\nimg, y = dataset[10]\n\nplt.figure(figsize=(8, 3))\nplt.title('Spectrogram')\nplt.xlabel('time')\nplt.ylabel('frequency')\nplt.imshow(img[0, 300:360]) # zooming in for dataset[10]\nplt.colorbar()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, name, *, pretrained=False):\n        \"\"\"\n        name (str): timm model name, e.g. tf_efficientnet_b2_ns\n        \"\"\"\n        super().__init__()\n\n        # Use timm\n        model = timm.create_model(name, pretrained=pretrained, in_chans=2)\n\n        clsf = model.default_cfg['classifier']\n        n_features = model._modules[clsf].in_features\n        model._modules[clsf] = nn.Identity()\n\n        self.fc = nn.Linear(n_features, 1)\n        self.model = model\n\n    def forward(self, x):\n        x = self.model(x)\n        x = self.fc(x)\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_fn(valid_loader, model, criterion, device, compute_score=True):\n    \n    tb = time.time() \n    model.eval() # switch to evaluation mode\n    preds = []\n    y_all = []\n    running_loss = 0\n    count = 0\n    \n    pbar = tqdm(valid_loader, total=len(valid_loader))\n    pbar.set_description(\"Validation\")\n    \n    for images, labels in pbar:\n        images = images.to(device)\n        labels = labels.to(device)\n        # compute loss\n        with torch.no_grad():\n            y_preds = model(images)\n        loss = criterion(y_preds.view(-1), labels)\n        running_loss += loss.item()*labels.shape[0]\n        count += 1\n        # record accuracy\n        y_all.append(labels.cpu().detach().numpy())\n        preds.append(y_preds.sigmoid().to('cpu').numpy())\n    \n    del loss, images, labels, y_preds\n    \n    y_ground = np.concatenate(y_all)\n    y_pred = np.concatenate(preds)\n    score = get_score(y_ground, y_pred) if compute_score else None \n    val_loss = running_loss/count\n    \n    val_dict = {'loss': val_loss,\n               'score': score,\n               'y': y,\n               'y_pred': y_pred,\n                'time': (time.time() - tb)/60\n               }\n    \n    return val_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = Model(CFG.model_name, pretrained=False)\n# submit = pd.read_csv(di + '/sample_submission.csv')\n# filename = f'/kaggle/input/g2net-train/model_0.pytorch'\n# model.to(device)\n# model.load_state_dict(torch.load(filename, map_location=device))\n# model.eval()\n# criterion = get_criterion()\n\n# # Predict\n# dataset_test = Dataset('test', submit)\n# loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=64,\n#                                             num_workers=CFG.num_workers, pin_memory=True)\n\n# test = valid_fn(loader_test, model, criterion, device, compute_score=False)\n# submit['target'] = test['y_pred']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model(CFG.model_name, pretrained=False)\nsubmit = pd.read_csv(di + '/sample_submission.csv')\ntest_res = []\n\nfor i in range(2):\n    filename = f'/kaggle/input/g2net-train/model{i}.pytorch'\n    model.to(device)\n    model.load_state_dict(torch.load(filename, map_location=device))\n    model.eval()\n    criterion = get_criterion()\n\n    # Predict\n    dataset_test = Dataset('test', submit)\n    loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=64,\n                                              num_workers=CFG.num_workers, pin_memory=True)\n\n    test = valid_fn(loader_test, model, criterion, device, compute_score=False)\n    test_res.append(test['y_pred'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res1 = 0\nfor i in range(2):\n    res1 += test_res[i]\nres1 /= 2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit['target'] = res1\nsubmit.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submit.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submit.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}