{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":37077,"databundleVersionId":4333111},{"sourceType":"datasetVersion","sourceId":4522512,"datasetId":2642879,"databundleVersionId":4583205}],"dockerImageVersionId":30302,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Acknowledgement \n\n## This code is inference code, model was trained with resources supported by AIN DAO.\n### - [proposal link](https://dao.ainetwork.ai/t/proposal-kaggle-competition-g2net-detecting-continuous-gravitational-waves/83) approved by the AIN DAO.\n\n<br/>\n\n## I would like to thank the AIN DAO for being able to conduct large-scale experiments with A100 machine\n\n\n## - Discord link: https://discord.gg/AnAmfWmdT2\n## - Discource link: https://dao.ainetwork.ai/\n\n\n<div class=\"alert alert-block alert-danger\" style=\"text-align:center; font-size:20px;\">\n    ❤️ Don't forget to join <a href=\"https://discord.gg/AnAmfWmdT2\">discord</a> if you find this notebook usefull!  ❤️\n</div>\n","metadata":{}},{"cell_type":"code","source":"!pip install timm","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc, glob, os\nfrom concurrent.futures import ProcessPoolExecutor\n\nimport h5py\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom scipy.stats import norm\nfrom timm import create_model\n\ndef normalize(X):\n    X = (X[..., None].view(X.real.dtype) ** 2).sum(-1)\n    POS = int(X.size * 0.99903)\n    EXP = norm.ppf((POS + 0.4) / (X.size + 0.215))\n    scale = np.partition(X.flatten(), POS, -1)[POS]\n    X /= scale / EXP.astype(scale.dtype) ** 2\n    return X\ndef dataload(filepath):\n    astime = np.full([2, 360, 5760], np.nan, dtype=np.float32)\n    with h5py.File(filepath, \"r\") as f:\n        fid, _ = os.path.splitext(os.path.split(filepath)[1])\n        HT = (np.asarray(f[fid][\"H1\"][\"timestamps_GPS\"]) / 1800).round().astype(np.int64)\n        LT = (np.asarray(f[fid][\"L1\"][\"timestamps_GPS\"]) / 1800).round().astype(np.int64)\n        MIN = min(HT.min(), LT.min()); HT -= MIN; LT -= MIN\n        H1 = normalize(np.asarray(f[fid][\"H1\"][\"SFTs\"], np.complex128))\n        valid = HT < 5760; astime[0][:, HT[valid]] = H1[:, valid]\n        L1 = normalize(np.asarray(f[fid][\"L1\"][\"SFTs\"], np.complex128))\n        valid = LT < 5760; astime[1][:, LT[valid]] = L1[:, valid]\n    gc.collect()\n    return fid, astime, H1.mean(), L1.mean()\nclass LargeKernel_debias(nn.Conv2d):\n    def forward(self, input: torch.Tensor):\n        finput = input.flatten(0, 1)[:, None]\n        target = abs(self.weight)\n        target = target / target.sum((-1, -2), True)\n        joined_kernel = torch.cat([self.weight, target], 0)\n        reals = target.new_zeros(\n            [1, 1] + [s + p * 2 for p, s in zip(self.padding, input.shape[-2:])]\n        )\n        reals[\n            [slice(None)] * 2 + [slice(p, -p) if p != 0 else slice(None) for p in self.padding]\n        ].fill_(1)\n        output, power = torch.nn.functional.conv2d(\n            finput, joined_kernel, padding=self.padding\n        ).chunk(2, 1)\n        ratio = torch.div(*torch.nn.functional.conv2d(reals, joined_kernel).chunk(2, 1))\n        output.sub_(power.mul_(ratio))\n        return output.unflatten(0, input.shape[:2]).flatten(1, 2)\ndef preprocess(num, input, H1, L1):\n    input = torch.from_numpy(input).to(\"cuda\", non_blocking=True)\n    rescale = torch.tensor([[H1, L1]]).to(\"cuda\", non_blocking=True)\n    tta = (\n        torch.randn(\n            [num, *input.shape, 2], device=input.device, dtype=torch.float32\n        )\n        .square_()\n        .sum(-1)\n    )\n    tta *= rescale[..., None, None] / 2\n    valid = ~torch.isnan(input); tta[:, valid] = input[valid].float()\n    return tta\ndef get_model(path):\n    model = create_model(\n        \"tf_efficientnetv2_b0\",\n        in_chans=32,\n        num_classes=2,\n    )\n    state_dict = torch.load(path)\n    C, _, H, W = state_dict[\"conv_stem.2.weight\"].shape\n    model.conv_stem = nn.Sequential(\n        nn.Identity(),\n        nn.AvgPool2d((1, 9), (1, 8), (0, 4), count_include_pad=False),\n        LargeKernel_debias(1, C, [H, W], 1, [H//2, W//2], 1, 1, False),\n        model.conv_stem,\n    )\n    model.load_state_dict(state_dict)\n    model.cuda().eval()\n    return model\n@torch.no_grad()\ndef inference(model, path):\n    file_path = glob.glob(os.path.join(path, \"*.hdf5\"))\n    FID, RES = [], []\n    with ProcessPoolExecutor(2) as pool:\n        for fid, input, H1, L1 in pool.map(dataload, sorted(file_path)):\n            tta = preprocess(64, input, H1, L1)\n            FID += [fid]\n            RES += [model(tta).softmax(-1)[..., 1].mean(0)]\n    return FID, torch.stack(RES, 0).cpu().float().numpy()\nif __name__ == \"__main__\":\n    model = get_model(\"../input/g2netdetectingcontinuousgravitationalwavesv0/group0/model_best.pth\")\n    fid, infer = inference(\n        model, \"../input/g2net-detecting-continuous-gravitational-waves/test\"\n    )\n    result = pd.DataFrame.from_dict({\"id\": fid, \"target\": infer})\n    result.to_csv(\"submission.csv\", index=False)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-17T09:39:29.513984Z","iopub.execute_input":"2022-11-17T09:39:29.514346Z","iopub.status.idle":"2022-11-17T09:40:05.420739Z","shell.execute_reply.started":"2022-11-17T09:39:29.514316Z","shell.execute_reply":"2022-11-17T09:40:05.419427Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# What signals will the model capture?\n\nHuge kernels can focus on shape rather than texture\n\nInspiration from: [Scaling Up Your Kernels to 31x31: Revisiting Large Kernel Design in CNNs](https://arxiv.org/abs/2203.06717)\n\n<img src=\"https://user-images.githubusercontent.com/16400591/162912476-7514efc2-a557-40f7-9af9-bdb891525cf3.png\">","metadata":{}},{"cell_type":"markdown","source":"# Signal filter visualization\n\n\nI get the impression that filters have a shape similar to the sobel operator.\n\nSobel–Feldman operator:\n\n<img src=\"https://wikimedia.org/api/rest_v1/media/math/render/svg/5b5d838b3f7e19843e8af8e9e0afdc5fd44bc7ba\">","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfor weight in model.conv_stem[2].weight.flatten(0, 1).detach().cpu().numpy():\n    plt.matshow(weight, cmap=\"seismic\", vmin=-0.7, vmax=0.7)\n    plt.title(f\"{weight.min():.3f}/{weight.mean():.3f}/{np.median(weight):.3f}/{weight.max():.3f}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-17T08:02:54.926091Z","iopub.execute_input":"2022-11-17T08:02:54.926475Z","iopub.status.idle":"2022-11-17T08:02:59.804104Z","shell.execute_reply.started":"2022-11-17T08:02:54.926444Z","shell.execute_reply":"2022-11-17T08:02:59.802915Z"},"trusted":true},"outputs":[],"execution_count":null}]}