{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\n\n# Assuming your github token is available in kaggle secrets as `gh_token`\ngh_token = user_secrets.get_secret(\"gh_token\")\n\nrepo = f\"https://nkitsaini:{gh_token}@github.com/nkitsaini/g2net.git\"\n\n!pip3 install \"git+{repo}\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-12T07:34:41.456972Z","iopub.execute_input":"2022-11-12T07:34:41.457866Z","iopub.status.idle":"2022-11-12T07:35:08.540307Z","shell.execute_reply.started":"2022-11-12T07:34:41.457779Z","shell.execute_reply":"2022-11-12T07:35:08.539078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import g2net\nfrom pathlib import Path\nPath(g2net.__file__)\nfrom g2net import *","metadata":{"execution":{"iopub.status.busy":"2022-11-12T07:35:08.542813Z","iopub.execute_input":"2022-11-12T07:35:08.543704Z","iopub.status.idle":"2022-11-12T07:35:13.596344Z","shell.execute_reply.started":"2022-11-12T07:35:08.543657Z","shell.execute_reply":"2022-11-12T07:35:13.595169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run_tensorboard()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g2net.run_notebook_cmd('train.py')","metadata":{"execution":{"iopub.status.busy":"2022-11-12T07:35:13.598211Z","iopub.execute_input":"2022-11-12T07:35:13.599253Z","iopub.status.idle":"2022-11-12T07:35:13.604891Z","shell.execute_reply.started":"2022-11-12T07:35:13.599211Z","shell.execute_reply":"2022-11-12T07:35:13.603756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!python3 /opt/conda/lib/python3.7/site-packages/notebooks/train.py","metadata":{"execution":{"iopub.status.busy":"2022-11-12T07:35:13.607656Z","iopub.execute_input":"2022-11-12T07:35:13.608064Z","iopub.status.idle":"2022-11-12T07:35:13.61613Z","shell.execute_reply.started":"2022-11-12T07:35:13.608027Z","shell.execute_reply":"2022-11-12T07:35:13.61498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%\nimport g2net\nimport importlib\n\nimportlib.reload(g2net.prelude)\nimportlib.reload(g2net.dataset)\nimportlib.reload(g2net)\n\nfrom g2net import *\n\n# \"\"\"\n# - [ ] \n\n# \"\"\"\n\n# %%\n\nsample_entry = G2NetTrainDataset.default()[0][0]\n\na: np.ndarray = sample_entry.h1.sft\nten = Tensor(a)\npad_size = 5000 - ten.shape[1]\nb = F.pad(ten, (0, pad_size), \"constant\", 0)\n\nb.shape\n# Tensor(a).to_padded_tensor(0, (360, 5000))\n#%%\n\nMAX_ENTRY_LEN = 5000\ndef _pad_sft(sft):\n    pad_size = MAX_ENTRY_LEN - sft.shape[1]\n    return F.pad(sft, (0, pad_size), 'constant', 0)\n\ndef _pad_ts(sft):\n    pad_size = MAX_ENTRY_LEN - sft.shape[0]\n    return F.pad(sft, (0, 0, 0, pad_size), 'constant', 0)\n\n#                                h1 sft, h1 ts, l1 sft, l1 ts, freq \ndef transform(val: G2Entry) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor]:\n    hs = _pad_sft(Tensor(val.h1.sft))\n    ht = _pad_ts(Tensor(val.h1.ts))\n    ls = _pad_sft(Tensor(val.l1.sft))\n    lt = _pad_ts(Tensor(val.l1.ts))\n    freq = Tensor(val.freq)\n    return (hs, ht, ls, lt, freq)\n\n\n(hs, ht, ls, lt, freq) =  transform(sample_entry)\nprint(hs.shape)\nprint(ht.shape)\nprint(ls.shape)\nprint(lt.shape)\nprint(freq.shape)\n#%%\npadded_dataset = G2NetTrainDataset.default(transform)\n\nprint(padded_dataset[0][0][0].shape)\nprint(padded_dataset[0][0][1].shape)\nprint(padded_dataset[0][0][2].shape)\nprint(padded_dataset[0][0][3].shape)\nprint(padded_dataset[0][0][4].shape)\n#%%\ntrain_size = int(len(padded_dataset) * 0.9)\nval_size = len(padded_dataset) - train_size\ntrain_data, val_data = tdata.random_split(padded_dataset, (train_size, val_size))\n\n\ntrain_loader = DataLoader(train_data, batch_size=8, sampler=tdata.RandomSampler(train_data))\nval_loader = DataLoader(val_data)\n\n#%%\nsample_batch = next(iter(train_loader))\n\n((hs, ht, ls, lt, f), l) = sample_batch\nhs.shape\nht.transpose(1, 2).shape\nf.squeeze(2).shape\n\n# b: Tensor = train_data[0][0][1]\n# c: Tensor = train_data[0][0][0]\n# f: Tensor = train_data[0][0][4]\n\n# b.transpose(0, 1).shape, c.shape\n# torch.cat((b.transpose(0, 1), c), 0).shape\n# f = f.squeeze()\n# f = f.unsqueeze(0)\n# nn.Linear(360, 5000)(f).shape\n# b = padded_dataset[0]\n# b[0][0].shape\n# b.size()\n#%%\n\n\"\"\"\nsft -> 2dConv -> (360, 5000)  -> (360, 5000) -> \nts -> 1dConv ->  (5000, 1) -> flip (1, 5000)\nsft -> 2dConv ->                (360, 5000) -> \nts -> 1dConv ->  (5000, 1) -> flip (1, 5000)\nfreq -> untouched   (360) nn      (1, 5000)\nMIXED -> (623, 5000) -> \n\"\"\"\n#%%\n# from tqdm.auto import tqdm\n# # from tqdm import tqdm\n# # # from tqdm.notebook import tqdm\n# # import time\n\n# for i in tqdm(range(10)):\n#     time.sleep(0.2)\n\n#%%\n\nclass MShape(nn.Module):\n    def __init__(self) -> None:\n        super().__init__()\n    def forward(self, x):\n        print(x.shape)\n        return\n\nclass Model(pl.LightningModule):\n    def __init__(self) -> None:\n        super().__init__()\n        self.freq_nn = nn.Linear(360, 5000)\n        self.layers = nn.Sequential(\n            nn.Conv2d(1, 3, 3),\n            nn.ReLU(),\n            nn.MaxPool2d(3),\n            nn.Dropout(),\n            nn.ReLU(),\n\n            nn.Conv2d(3, 3, 5),\n            nn.ReLU(),\n            nn.MaxPool2d(5),\n            nn.Dropout(),\n            nn.ReLU(),\n\n            nn.Conv2d(3, 5, 16),\n            nn.ReLU(),\n            nn.MaxPool2d(5),\n            # MShape(),\n            nn.LayerNorm([5, 6, 63]),\n\n            nn.Flatten(),\n            nn.Dropout(),\n\n            nn.Linear(5*6*63, 1024*8),\n            nn.ReLU(),\n            nn.Linear(1024*8, 16),\n            nn.ReLU(),\n            nn.Dropout(),\n            nn.Linear(16, 1),\n            nn.Sigmoid()\n        )\n    \n    def forward(self, batch):\n        (hs, ht, ls, lt, freq) = batch\n        freq = freq.squeeze(2)\n        freq = self.freq_nn(freq)\n        freq = freq.unsqueeze(1)\n        ht = ht.transpose(1, 2)\n        lt = lt.transpose(1, 2)\n        x = torch.cat((hs, ht, ls, lt, freq), 1)\n        # Add layer for conv2d\n        x = x.unsqueeze(1)\n        x =  self.layers(x)\n        return x.squeeze(1)\n    \n    def configure_optimizers(self) -> Any:\n        return optim.Adam(self.parameters())\n    \n    def training_step(self, batch, batch_idx: int):\n        x, y = batch\n        z = self(x)\n\n        # print(z.type(), y.type(torch.FloatTensor))\n        # return\n        loss =  F.binary_cross_entropy(z, y.type(torch.float))\n        self.log('train_loss', loss)\n        return loss\n\n    def test_step(self, batch, batch_idx: int):\n        x, y = batch\n        z = self(x)\n\n        loss =  F.binary_cross_entropy(z, y.type(torch.float))\n        # loss =  F.binary_cross_entropy(z, y)\n        self.log('test_loss', loss)\n        return loss\n\n    def validation_step(self, batch, batch_idx: int):\n        x, y = batch\n        z = self(x)\n\n        loss =  F.binary_cross_entropy(z, y.type(torch.float))\n        # loss =  F.binary_cross_entropy(z, y)\n        self.log('val_loss', loss)\n        return loss\n\n\nmodel = Model()\n\naccelerator = 'gpu' if torch.cuda.is_available() else None\ndevices = 1 if torch.cuda.is_available() else None\n\ntrainer = pl.Trainer(callbacks=[ProgressCallback()], accelerator=accelerator, devices=devices, max_epochs=20)\ntrainer.fit(model, train_loader, val_loader)\n\n# model.training_step(sample_batch, 0)","metadata":{"execution":{"iopub.status.busy":"2022-11-12T08:04:37.64256Z","iopub.execute_input":"2022-11-12T08:04:37.643033Z","iopub.status.idle":"2022-11-12T08:44:16.068855Z","shell.execute_reply.started":"2022-11-12T08:04:37.64299Z","shell.execute_reply":"2022-11-12T08:44:16.067595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = Tensor()","metadata":{"execution":{"iopub.status.busy":"2022-11-12T06:48:10.304331Z","iopub.execute_input":"2022-11-12T06:48:10.304772Z","iopub.status.idle":"2022-11-12T06:48:10.310602Z","shell.execute_reply.started":"2022-11-12T06:48:10.304733Z","shell.execute_reply":"2022-11-12T06:48:10.309505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -sch *","metadata":{"execution":{"iopub.status.busy":"2022-11-12T08:44:40.987951Z","iopub.execute_input":"2022-11-12T08:44:40.988365Z","iopub.status.idle":"2022-11-12T08:44:42.005518Z","shell.execute_reply.started":"2022-11-12T08:44:40.988331Z","shell.execute_reply":"2022-11-12T08:44:42.004094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a.get_device()","metadata":{"execution":{"iopub.status.busy":"2022-11-12T06:48:17.878144Z","iopub.execute_input":"2022-11-12T06:48:17.878851Z","iopub.status.idle":"2022-11-12T06:48:17.88747Z","shell.execute_reply.started":"2022-11-12T06:48:17.878813Z","shell.execute_reply":"2022-11-12T06:48:17.88617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a.cuda().type(torch.float)","metadata":{"execution":{"iopub.status.busy":"2022-11-12T06:48:52.769169Z","iopub.execute_input":"2022-11-12T06:48:52.769663Z","iopub.status.idle":"2022-11-12T06:48:52.777498Z","shell.execute_reply.started":"2022-11-12T06:48:52.769617Z","shell.execute_reply":"2022-11-12T06:48:52.776415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}