{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip3 install -U pytorch-optimizer\n!pip install -U \"anc2vec @ git+https://github.com/aedera/anc2vec.git\"","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:29:45.679364Z","iopub.execute_input":"2023-07-05T03:29:45.679619Z","iopub.status.idle":"2023-07-05T03:30:19.115995Z","shell.execute_reply.started":"2023-07-05T03:29:45.679594Z","shell.execute_reply":"2023-07-05T03:30:19.114694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport time\nimport copy\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, random_split\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, OneCycleLR, CosineAnnealingLR\nfrom torchmetrics.classification import MultilabelF1Score\nfrom torchmetrics.classification import MultilabelAccuracy\nfrom pytorch_optimizer import Ranger, SoftF1Loss\n\nfrom sklearn.metrics import roc_auc_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-05T03:30:19.129005Z","iopub.execute_input":"2023-07-05T03:30:19.131479Z","iopub.status.idle":"2023-07-05T03:30:33.498531Z","shell.execute_reply.started":"2023-07-05T03:30:19.131436Z","shell.execute_reply":"2023-07-05T03:30:33.497499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\"root_data\": '/kaggle/input/cafa-5-protein-function-prediction',\n          \"root_embed\": '/kaggle/input/23468234',\n          \"exp_name\": 'all, 600 label, 250 iter, sigmoid cosine, seed : 42',\n          \"n_labels\": 600,\n          \"seeds\": [42],\n          \"min_ia\": 1,\n          \"epoch\": 250,\n          \"lr\": 1e-3,\n          \"batch_size\": 256,\n          \"n_accumulate\": 1,\n          \"loss\": 'cce', # 'cce', 'sigmoidf1' 'softf1'\n          \"scheduler\": 'cosine', # 'cosine', 'onecycle'\n          \"device\": torch.device('cuda' if torch.cuda.is_available() else 'cpu')}","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:30:33.500144Z","iopub.execute_input":"2023-07-05T03:30:33.500516Z","iopub.status.idle":"2023-07-05T03:30:33.536526Z","shell.execute_reply.started":"2023-07-05T03:30:33.500481Z","shell.execute_reply":"2023-07-05T03:30:33.535233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"## Experiment: \")\nprint(CONFIG['exp_name'])","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:30:33.540426Z","iopub.execute_input":"2023-07-05T03:30:33.540777Z","iopub.status.idle":"2023-07-05T03:30:33.549602Z","shell.execute_reply.started":"2023-07-05T03:30:33.540747Z","shell.execute_reply":"2023-07-05T03:30:33.54866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    os.environ['PYTHONHASHSEED'] = str(seed)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:30:33.55261Z","iopub.execute_input":"2023-07-05T03:30:33.553061Z","iopub.status.idle":"2023-07-05T03:30:33.560474Z","shell.execute_reply.started":"2023-07-05T03:30:33.553022Z","shell.execute_reply":"2023-07-05T03:30:33.559043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get Train Terms & Load Embeddings ","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(os.path.join(CONFIG[\"root_data\"], 'Train', 'train_terms.tsv'), sep='\\t')\nIA = pd.read_csv(os.path.join(CONFIG[\"root_data\"], 'IA.txt'), sep='\\t', header=None, names=[\"term\", \"val\"])\nIA = IA[IA[\"val\"] > CONFIG[\"min_ia\"]].reset_index().drop([\"index\"], axis=1)\nIA = IA[\"term\"].values","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:30:33.561952Z","iopub.execute_input":"2023-07-05T03:30:33.562597Z","iopub.status.idle":"2023-07-05T03:30:37.043507Z","shell.execute_reply.started":"2023-07-05T03:30:33.562546Z","shell.execute_reply":"2023-07-05T03:30:37.04255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = np.load('/kaggle/input/4637427/train_ids_esm2_t36_3B_UR50D.npy')\nlabels = train_data\n\ntop_terms = labels.groupby(\"term\")[\"EntryID\"].count().sort_values(ascending=False)\nlabels_names = top_terms[:CONFIG[\"n_labels\"]].index.values\ntrain_labels_sub = labels[(labels.term.isin(labels_names)) & (labels.EntryID.isin(ids))]\nid_labels = train_labels_sub.groupby('EntryID')['term'].apply(list).to_dict()\n\ngo_terms_map = {label: i for i, label in enumerate(labels_names)}\nlabels_matrix = np.empty((len(ids), len(labels_names)))\n\nfor index, id in tqdm(enumerate(ids), total=len(ids)):\n    id_gos_list = id_labels[id]\n    temp = [go_terms_map[go] for go in labels_names if go in id_gos_list]\n    labels_matrix[index, temp] = 1\n    \nprint(labels_matrix)\nprint(labels_matrix.shape)\n\nnp.save(\"/kaggle/working/train_targets_top\"+str(CONFIG[\"n_labels\"])+\".npy\", np.array(labels_matrix))\n\ndel labels_matrix\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:30:37.044798Z","iopub.execute_input":"2023-07-05T03:30:37.045124Z","iopub.status.idle":"2023-07-05T03:32:12.419423Z","shell.execute_reply.started":"2023-07-05T03:30:37.045093Z","shell.execute_reply":"2023-07-05T03:32:12.418399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class ProteinDataset(Dataset):\n    def __init__(self, ids_source, embedding_source1, embedding_source2, target_source, train=True):\n        ids = np.load(ids_source)\n        if embedding_source2 is not None:\n            embeds1 = np.load(embedding_source1)\n            embeds2 = np.load(embedding_source2)\n            embeds = np.concatenate((embeds1, embeds2), axis=1)\n\n            del embeds1, embeds2\n            gc.collect()\n            \n            print(embeds.shape)\n        \n        else:\n            embeds = np.load(embedding_source1)\n        \n        self.train = train\n        \n        embeds_list = []\n        for l in range(embeds.shape[0]):\n            embeds_list.append(embeds[l,:])\n            \n        self.df = pd.DataFrame(data={\"EntryID\": ids, \"embed\" : embeds_list})\n        \n        if self.train:\n            target = np.load(target_source)\n            df_labels = pd.DataFrame({\"EntryID\": ids, \"target\": target.tolist()})\n            del target\n            gc.collect()\n            self.df = self.df.merge(df_labels, on=\"EntryID\")\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        embed = torch.tensor(self.df.iloc[index][\"embed\"] , dtype = torch.float32)\n        if self.train:\n            target = torch.tensor(self.df.iloc[index][\"target\"], dtype = torch.float32)\n            return {\n                'embed': embed,\n                'target': target,\n            }\n        else:\n            id = self.df.iloc[index][\"EntryID\"]\n            return { \n                'embed': embed,\n                'id' : id\n            }","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:32:12.420921Z","iopub.execute_input":"2023-07-05T03:32:12.42138Z","iopub.status.idle":"2023-07-05T03:32:12.434827Z","shell.execute_reply.started":"2023-07-05T03:32:12.421346Z","shell.execute_reply":"2023-07-05T03:32:12.433784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class LinearModel(nn.Module):\n    def __init__(self, embed_size, hidden_size1, hidden_size2, target_size, dropout=0.8, num_emb_layers=2):\n        super(LinearModel, self).__init__()\n        self.input_block = nn.Sequential(nn.LayerNorm(embed_size, eps=1e-6),\n                                         nn.Linear(embed_size, hidden_size1), \n                                         nn.LeakyReLU())\n\n        self.hidden_block1 = []\n        for i in range(num_emb_layers - 1):\n            self.hidden_block1.extend([nn.LayerNorm(hidden_size1, eps=1e-6), \n                                       nn.Dropout(0.15), \n                                       nn.Linear(hidden_size1, hidden_size1), \n                                       nn.LeakyReLU()])\n            \n            if i == num_emb_layers - 2:\n                self.hidden_block1.extend([nn.LayerNorm(hidden_size1, eps=1e-6)])\n        self.hidden_block1.extend([nn.LayerNorm(hidden_size1, eps=1e-6), \n                                   nn.Dropout(0.15), \n                                   nn.Linear(hidden_size1, hidden_size2)])\n        self.hidden_block1 = nn.Sequential(*self.hidden_block1)\n        \n        self.hidden_block2 = []\n        for i in range(num_emb_layers - 1):\n            self.hidden_block2.extend([nn.LayerNorm(hidden_size2, eps=1e-6), \n                                       nn.Dropout(0.15), \n                                       nn.Linear(hidden_size2, hidden_size2), \n                                       nn.LeakyReLU()])\n            \n            if i == num_emb_layers - 2:\n                self.hidden_block2.extend([nn.LayerNorm(hidden_size2, eps=1e-6)])\n        self.hidden_block2 = nn.Sequential(*self.hidden_block2)\n        \n        self.output_block = nn.Sequential(nn.LayerNorm(hidden_size2, eps=1e-6), \n                                          nn.Dropout(0.15), \n                                          nn.Linear(hidden_size2, target_size))\n        \n        self.dropout = nn.Dropout(dropout)\n        \n        for p in self.parameters():\n            if p.dim() > 1:\n                nn.init.xavier_uniform_(p)\n        \n    def forward(self, x):        \n        out = self.input_block(x)\n        out = self.dropout(self.hidden_block1(out))\n        out = self.dropout(self.hidden_block2(out))\n        out = self.output_block(out)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:32:12.436509Z","iopub.execute_input":"2023-07-05T03:32:12.437305Z","iopub.status.idle":"2023-07-05T03:32:12.453585Z","shell.execute_reply.started":"2023-07-05T03:32:12.437265Z","shell.execute_reply":"2023-07-05T03:32:12.452532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://github.com/gabriben/metrics-as-losses\nclass sigmoidF1(nn.Module):\n\n    def __init__(self, S = -1, E = 0):\n        super(sigmoidF1, self).__init__()\n        self.S = S\n        self.E = E\n\n    @torch.cuda.amp.autocast()\n    def forward(self, y_hat, y):\n        \n        y_hat = torch.sigmoid(y_hat)\n\n        b = torch.tensor(self.S)\n        c = torch.tensor(self.E)\n\n        sig = 1 / (1 + torch.exp(b * (y_hat + c)))\n\n        tp = torch.sum(sig * y, dim=0)\n        fp = torch.sum(sig * (1 - y), dim=0)\n        fn = torch.sum((1 - sig) * y, dim=0)\n\n        sigmoid_f1 = 2*tp / (2*tp + fn + fp + 1e-16)\n        cost = 1 - sigmoid_f1\n        macroCost = torch.mean(cost)\n\n        return macroCost","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:32:12.458126Z","iopub.execute_input":"2023-07-05T03:32:12.45845Z","iopub.status.idle":"2023-07-05T03:32:12.470592Z","shell.execute_reply.started":"2023-07-05T03:32:12.458423Z","shell.execute_reply":"2023-07-05T03:32:12.469399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def criterion(output, target):\n    if CONFIG['loss'] == \"softf1\":\n        loss = SoftF1Loss()\n    elif CONFIG['loss'] == \"cce\":\n        loss = nn.CrossEntropyLoss(label_smoothing=0.1)\n    elif CONFIG['loss'] == \"sigmoidf1\":\n        loss = sigmoidF1()\n    else:\n        raise NotImplementedError\n    return loss(output, target)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:32:12.472477Z","iopub.execute_input":"2023-07-05T03:32:12.472897Z","iopub.status.idle":"2023-07-05T03:32:12.486458Z","shell.execute_reply.started":"2023-07-05T03:32:12.472857Z","shell.execute_reply":"2023-07-05T03:32:12.485348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def f1score(output, target):\n    score = MultilabelF1Score(num_labels=CONFIG['n_labels']).to(CONFIG['device'])\n    return score(output, target)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:32:12.48797Z","iopub.execute_input":"2023-07-05T03:32:12.488953Z","iopub.status.idle":"2023-07-05T03:32:12.498308Z","shell.execute_reply.started":"2023-07-05T03:32:12.488914Z","shell.execute_reply":"2023-07-05T03:32:12.497329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, epoch, verbose):\n    model.train()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    running_score = 0.0\n    \n    #bar = tqdm(enumerate(dataloader), total=len(dataloader), leave=verbose)\n    for step, data in enumerate(dataloader):\n        embeds = data['embed'].to(CONFIG['device'])\n        targets = data['target'].to(CONFIG['device'])\n        batch_size = embeds.size(0)\n        \n        preds = model(embeds)\n        \n        score = f1score(preds, targets)\n        loss = criterion(preds, targets)\n        loss = loss / CONFIG['n_accumulate']\n        loss.backward()\n    \n        if (step + 1) % CONFIG['n_accumulate'] == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n            \n            if scheduler is not None:\n                scheduler.step(epoch_loss)\n                \n        running_loss += (loss.item() * batch_size)\n        running_score += (score.item() * batch_size)\n        dataset_size += batch_size\n        \n    epoch_loss = running_loss / dataset_size\n    epoch_score = running_score / dataset_size\n        \n#     bar.set_postfix(Epoch=epoch, \n#                     Train_Loss=epoch_loss,\n#                     LR=optimizer.param_groups[0]['lr'])\n        \n    # garbage collection\n    gc.collect()\n    \n    return epoch_loss, epoch_score","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:32:12.501329Z","iopub.execute_input":"2023-07-05T03:32:12.503823Z","iopub.status.idle":"2023-07-05T03:32:12.513913Z","shell.execute_reply.started":"2023-07-05T03:32:12.503794Z","shell.execute_reply":"2023-07-05T03:32:12.512887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\n\ndef valid_one_epoch(model, optimizer, dataloader, scheduler, epoch, verbose):\n    model.eval()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    running_score = 0.0\n    \n    #bar = tqdm(enumerate(dataloader), total=len(dataloader), leave=verbose)\n    for step, data in enumerate(dataloader):        \n        embeds = data['embed'].to(CONFIG['device'])\n        targets = data['target'].to(CONFIG['device'])\n        batch_size = embeds.size(0)\n        \n        preds = model(embeds)\n        \n        score = f1score(preds, targets)\n        loss = criterion(preds, targets)\n        loss = loss / CONFIG['n_accumulate']\n        \n        running_loss += (loss.item() * batch_size)\n        running_score += (score.item() * batch_size)\n        dataset_size += batch_size\n        \n    epoch_loss = running_loss / dataset_size\n    epoch_score = running_score / dataset_size\n        \n#     bar.set_postfix(Epoch=epoch, \n#                     Valid_Loss=epoch_loss,\n#                     LR=optimizer.param_groups[0]['lr'])   \n    \n    gc.collect()\n    \n    return epoch_loss, epoch_score","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:32:12.515481Z","iopub.execute_input":"2023-07-05T03:32:12.516435Z","iopub.status.idle":"2023-07-05T03:32:12.529727Z","shell.execute_reply.started":"2023-07-05T03:32:12.516389Z","shell.execute_reply":"2023-07-05T03:32:12.528695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, train_loader, val_loader, num_epochs, verbose=True):\n    train_score = []\n    val_score = []\n    train_loss = []\n    val_loss = []\n    \n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_epoch_loss = np.inf                                                                                             \n    \n    for epoch in range(num_epochs):\n        if verbose:\n            print(f\"Epoch: {epoch}\")\n            \n        gc.collect()\n        train_epoch_loss, train_epoch_score = train_one_epoch(model=model, \n                                                              optimizer=optimizer, \n                                                              scheduler=scheduler, \n                                                              dataloader=train_loader,  \n                                                              epoch=epoch,\n                                                              verbose=verbose)\n        train_score.append(train_epoch_score)\n        train_loss.append(train_epoch_loss)\n        \n        if val_loader is not None:                                                                      \n            val_epoch_loss, val_epoch_score = valid_one_epoch(model=model, \n                                                              optimizer=optimizer,\n                                                              dataloader=val_loader, \n                                                              scheduler=scheduler,\n                                                              epoch=epoch,\n                                                              verbose=verbose)\n\n            val_score.append(val_epoch_score)\n            val_loss.append(val_epoch_loss)\n            if verbose:\n                print(f\"Train loss: {train_epoch_loss} Validation loss: {val_epoch_loss}\")\n                print(f\"Train score: {train_epoch_score} Validation score: {val_epoch_score}\")\n\n            if val_epoch_loss <= best_epoch_loss:\n                if verbose:\n                    print(f\"Validation Loss Improved ({best_epoch_loss} ---> {val_epoch_loss})\")\n\n                best_epoch_loss = val_epoch_loss\n                best_model_wts = copy.deepcopy(model.state_dict())\n\n            print()\n        \n        else:\n            val_score = None\n            val_loss = None\n            if verbose:\n                print(f\"Train loss: {train_epoch_loss}\")\n                print(f\"Train score: {train_epoch_score}\")\n            \n            if train_epoch_loss <= best_epoch_loss:\n                if verbose:\n                    print(f\"Score Improved ({best_epoch_loss} ---> {train_epoch_loss})\")\n                best_model_wts = copy.deepcopy(model.state_dict())\n                                                                                                  \n        end = time.time()\n                                                                                                  \n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    \n    print(\"Best Score: {:.4f}\".format(best_epoch_loss))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, train_score, train_loss, val_score, val_loss                                                                         ","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:32:12.531422Z","iopub.execute_input":"2023-07-05T03:32:12.532184Z","iopub.status.idle":"2023-07-05T03:32:12.551029Z","shell.execute_reply.started":"2023-07-05T03:32:12.532135Z","shell.execute_reply":"2023-07-05T03:32:12.549963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run Training","metadata":{}},{"cell_type":"code","source":"def run_folds(folds, data='all', save_output=True):\n    num_labels = CONFIG['n_labels']\n    \n    print(\"## Loading Dataset\")\n    dataset = ProteinDataset(ids_source='/kaggle/input/4637427/train_ids_esm2_t36_3B_UR50D.npy',\n                             embedding_source1='/kaggle/input/23468234/train_embeds_esm2_t33_650M_UR50D.npy',\n                             embedding_source2=None,\n                             target_source=f\"/kaggle/working/train_targets_top{num_labels}.npy\",\n                             train=True)\n    train_losses = []\n    train_scores = []\n    val_losses = []\n    val_scores = []\n    \n    print(\"## Running Training\")\n    \n    for seed in folds:\n        set_seed(seed)\n        print(\"#### Seed: \", seed, \"####\")\n        print()\n        if data != 'all':\n            generator = torch.Generator().manual_seed(seed)\n            train_dataset, val_dataset = random_split(dataset, \n                                                      lengths = [int(len(dataset)*0.9), \n                                                                 len(dataset)-int(len(dataset)*0.9)],\n                                                      generator=generator)\n\n            train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=CONFIG['batch_size'], num_workers=2, shuffle=True)\n            val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=CONFIG['batch_size'], num_workers=2, shuffle=False)\n            \n        else:\n            train_dataloader = torch.utils.data.DataLoader(dataset, batch_size=CONFIG['batch_size'], num_workers=2, shuffle=True)\n            val_dataloader = None\n            \n        model = LinearModel(embed_size=2560, \n                            hidden_size1=1912,\n                            hidden_size2=1024,\n                            target_size=CONFIG['n_labels'], \n                            dropout=0.8, \n                            num_emb_layers=3).to(CONFIG['device'])\n\n        optimizer = Ranger(model.parameters(), lr=CONFIG['lr']) \n        \n        if CONFIG['scheduler'] == 'cosine':\n            scheduler = CosineAnnealingLR(optimizer, (CONFIG['epoch'] * 501))\n        elif CONFIG['scheduler'] == 'onecycle':\n            scheduler = OneCycleLR(optimizer, max_lr=CONFIG['lr'], total_steps=CONFIG['epoch'] * 501)\n\n        model, train_score, train_loss, val_score, val_loss = run_training(model=model, \n                                                                           optimizer=optimizer, \n                                                                           scheduler=None, \n                                                                           train_loader=train_dataloader, \n                                                                           val_loader=val_dataloader,\n                                                                           num_epochs=CONFIG['epoch'], \n                                                                           verbose=True)\n        \n        train_losses.append(train_loss)\n        train_scores.append(train_score)\n        val_losses.append(val_loss)\n        val_scores.append(val_score)\n        \n        if save_output:\n            root_save = \"/kaggle/working/\"\n            os.makedirs(os.path.dirname(f\"{root_save}/model_{seed}.pt\"), exist_ok=True)\n            torch.save(model.state_dict(), f\"{root_save}/model_{seed}.pt\")\n        \n        del model, train_dataloader, val_dataloader, train_score, train_loss, val_score, val_loss\n        _ = gc.collect()\n        \n    return train_losses, train_scores, val_losses, val_scores","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:32:12.554346Z","iopub.execute_input":"2023-07-05T03:32:12.554785Z","iopub.status.idle":"2023-07-05T03:32:12.572822Z","shell.execute_reply.started":"2023-07-05T03:32:12.554755Z","shell.execute_reply":"2023-07-05T03:32:12.571744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_losses, train_scores, val_losses, val_scores = run_folds(folds=CONFIG['seeds'],\n                                                               data=\"part\",\n                                                               save_output=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:32:12.574443Z","iopub.execute_input":"2023-07-05T03:32:12.574806Z","iopub.status.idle":"2023-07-05T03:34:30.081072Z","shell.execute_reply.started":"2023-07-05T03:32:12.574773Z","shell.execute_reply":"2023-07-05T03:34:30.080088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_losses[0])\nif val_losses[0] is not None:\n    print(val_losses[0])","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:34:30.083137Z","iopub.execute_input":"2023-07-05T03:34:30.083935Z","iopub.status.idle":"2023-07-05T03:34:30.090661Z","shell.execute_reply.started":"2023-07-05T03:34:30.083882Z","shell.execute_reply":"2023-07-05T03:34:30.089665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure()\nplt.plot(train_losses[0])\nif val_losses[0] is not None:\n    plt.plot(val_losses[0])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:34:30.092283Z","iopub.execute_input":"2023-07-05T03:34:30.093012Z","iopub.status.idle":"2023-07-05T03:34:30.40875Z","shell.execute_reply.started":"2023-07-05T03:34:30.092973Z","shell.execute_reply":"2023-07-05T03:34:30.407655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.plot(train_scores[0])\nif val_scores[0] is not None:\n    plt.plot(val_scores[0])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:34:30.411791Z","iopub.execute_input":"2023-07-05T03:34:30.412149Z","iopub.status.idle":"2023-07-05T03:34:30.666562Z","shell.execute_reply.started":"2023-07-05T03:34:30.41212Z","shell.execute_reply":"2023-07-05T03:34:30.665583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.remove(\"/kaggle/working/train_targets_top\"+str(CONFIG[\"n_labels\"])+\".npy\")","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:34:30.66823Z","iopub.execute_input":"2023-07-05T03:34:30.668588Z","iopub.status.idle":"2023-07-05T03:34:30.692178Z","shell.execute_reply.started":"2023-07-05T03:34:30.668553Z","shell.execute_reply":"2023-07-05T03:34:30.691167Z"},"trusted":true},"execution_count":null,"outputs":[]}]}