{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q pytorch-lightning-bolts\n!pip install -q command","metadata":{"execution":{"iopub.status.busy":"2022-10-31T14:48:43.378844Z","iopub.execute_input":"2022-10-31T14:48:43.379851Z","iopub.status.idle":"2022-10-31T14:49:09.417818Z","shell.execute_reply.started":"2022-10-31T14:48:43.379743Z","shell.execute_reply":"2022-10-31T14:49:09.416588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport os\nfrom glob import glob\nimport copy\nimport time\nimport math\nimport command\nimport random\n\nimport cv2\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nmpl.rcParams['figure.figsize'] = 12, 8\n\nfrom skimage import img_as_ubyte\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import Dataset, DataLoader\nimport pytorch_lightning as pl\nimport pl_bolts as pb\n\ndef seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nmeans = [0.4760, 0.0723, 0.1412, 0.0362, 0.0535, 0.0802, 0.1372, 0.1947]\n    \ndef competiton_loss_row_norm(y_hat, y, adjusted=1, with_logits=0):\n    \n    device = y_hat.device\n    \n    if with_logits:\n        loss_fn = nn.BCEWithLogitsLoss(reduction=\"none\")\n    else:\n        loss_fn = nn.BCELoss(reduction=\"none\")\n    \n    competition_weights = {\n    '-' : torch.tensor([7, 1, 1, 1, 1, 1, 1, 1], dtype=torch.float, device=device),\n    '+' : torch.tensor([14, 2, 2, 2, 2, 2, 2, 2], dtype=torch.float, device=device) * 1,\n    }\n    \n    if adjusted:\n        for _ in range(8):\n            competition_weights['+'][_] *= means[_] / torch.mean(y[:, _])\n    \n    loss = loss_fn(y_hat, y)\n    weights = y * competition_weights['+'] + (1 - y) * competition_weights['-']\n    loss = (loss * weights).sum(axis=1)\n    w_sum = weights.sum(axis=1)\n    loss = torch.div(loss, w_sum)\n    return loss.mean()\n\ndef sigmoid(x):\n    return 1 / (1 + np.exp(-x))","metadata":{"execution":{"iopub.status.busy":"2022-10-31T14:50:56.208061Z","iopub.execute_input":"2022-10-31T14:50:56.208849Z","iopub.status.idle":"2022-10-31T14:50:56.223268Z","shell.execute_reply.started":"2022-10-31T14:50:56.208812Z","shell.execute_reply":"2022-10-31T14:50:56.222274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DIR = '/kaggle/input/'\n\ntrue = pd.read_csv(f'{DIR}/rsna-2022-cervical-spine-fracture-detection/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-10-31T14:50:07.589145Z","iopub.execute_input":"2022-10-31T14:50:07.589609Z","iopub.status.idle":"2022-10-31T14:50:07.612564Z","shell.execute_reply.started":"2022-10-31T14:50:07.589567Z","shell.execute_reply":"2022-10-31T14:50:07.611139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = 1 # CHANGE THIS TO 0 IF WANT TO ACTUALLY TRAIN","metadata":{"execution":{"iopub.status.busy":"2022-10-31T14:49:56.818865Z","iopub.execute_input":"2022-10-31T14:49:56.819301Z","iopub.status.idle":"2022-10-31T14:49:56.828Z","shell.execute_reply.started":"2022-10-31T14:49:56.819262Z","shell.execute_reply":"2022-10-31T14:49:56.826995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    DDP = 0\n    DDP_INIT_DONE = 0\n    \n    SEED = 3407\n    SPLITS = 5\n    FOLD = 0\n    \n    TRN_BS = 4 #WAS ORIGINALLY TRAINED ON 128 GLOBAL BATCH SIZE\n    VAL_BS = 1\n    \n    EPOCHS = 2 if DEBUG else 12\n    LR = 1e-3\n    WARMUP_EPOCHS = 4\n    WARMUP_LR = 1e-6\n    \n    FOLDER = \"/kaggle/working/\"\n    NAME = \"b5_v5_pseudo_round1_seq\"\n    V = \"2\"\n    \n    DIM = 64\n    \n#pl.seed_everything(CFG.SEED)\nseed_everything(CFG.SEED)\nOUTPUT_FOLDER = f\"{CFG.FOLDER}/{CFG.NAME}_v{CFG.V}/\"\n\nCFG.cache_dir = OUTPUT_FOLDER + '/cache/'\n\nos.makedirs(OUTPUT_FOLDER, exist_ok=1)\nos.makedirs(CFG.cache_dir, exist_ok=1)","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:25:28.37378Z","iopub.execute_input":"2022-10-31T15:25:28.374166Z","iopub.status.idle":"2022-10-31T15:25:28.382785Z","shell.execute_reply.started":"2022-10-31T15:25:28.374137Z","shell.execute_reply":"2022-10-31T15:25:28.381784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_to_fold = np.load(f\"/{DIR}/b5-v5-t4-pseudo-round1-pseudo-to-fold/pseudo_to_fold.npy\", allow_pickle=1).item()\n\n#feats1 = glob('/mnt/md0/rsna_spine/seq_data/b5_v5_t5_pseudo_round1_data1/*_features.npy')\n#feats2 = glob('/mnt/md0/rsna_spine/seq_data/b5_v5_t5_pseudo_round1_data1_rest/*_features.npy')\n\nfeats1 = glob(f'{DIR}/b5-v5-t4-pseudo-round1-seq-data/b5_v5_t4_pseudo_round1_data1/*_features.npy')\nfeats2 = glob(f'{DIR}/b5-v5-t4-pseudo-round1-seq-data/b5_v5_t4_pseudo_round1_data1_rest/*_features.npy')\n\n#'''\nfeat_paths = np.array(feats2)#np.array(feats1 + feats2)\n\n#feat_paths = np.array(glob('/mnt/md0/rsna_spine/seq_data/data2/*_features.npy'))\n\nfolds = [*StratifiedKFold(n_splits=5).split(feat_paths, [true[true.StudyInstanceUID==f.split('/')[-1].split('_')[0]].values[0, 4] for f in feat_paths])]\n\ntrain_features, valid_features = feat_paths[folds[CFG.FOLD][0]], feat_paths[folds[CFG.FOLD][1]]\ntrain_features = np.array(train_features.tolist() + feats1)\n\n#'''\n\n'''\nfeat_paths = np.array(feats1)#np.array(feats1 + feats2)\n\n#feat_paths = np.array(glob('/mnt/md0/rsna_spine/seq_data/data2/*_features.npy'))\n\nfolds = [*StratifiedKFold(n_splits=5).split(feat_paths, [true[true.StudyInstanceUID==f.split('/')[-1].split('_')[0]].values[0, 4] for f in feat_paths])]\n\ntrain_features = feat_paths[folds[CFG.FOLD][0]]\nvalid_features = np.array(feats2)\n#'''\n\n#train_features, valid_features = train_test_split(feat_paths, test_size=0.2, random_state=42)\n\n#train_features = np.array(glob('/mnt/md0/rsna_spine/seq_data/data4_/*_features.npy'))\n#valid_features = np.array(glob('/mnt/md0/rsna_spine/seq_data/data1/*_features.npy'))\n\ntrain_features.shape, valid_features.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:25:29.291979Z","iopub.execute_input":"2022-10-31T15:25:29.292383Z","iopub.status.idle":"2022-10-31T15:25:29.74126Z","shell.execute_reply.started":"2022-10-31T15:25:29.292329Z","shell.execute_reply":"2022-10-31T15:25:29.740062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SpineDataset(Dataset):\n    def __init__(self, data, transforms=None, is_train=0):\n        self.data = data\n        self.is_train = is_train\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, i):\n        f = self.data[i]\n        pat = f.split('/')[-1].split('_')[0]\n        tgs = true[true.StudyInstanceUID==pat].values[0, 1:].astype(np.float32)\n        \n        feat = np.load(f)\n        \n        feat = np.mean(feat, 2)\n        return feat, tgs, f","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:25:29.743396Z","iopub.execute_input":"2022-10-31T15:25:29.743969Z","iopub.status.idle":"2022-10-31T15:25:29.751538Z","shell.execute_reply.started":"2022-10-31T15:25:29.74393Z","shell.execute_reply":"2022-10-31T15:25:29.750511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_loaders():\n    \n    train_features, valid_features = feat_paths[folds[CFG.FOLD][0]], feat_paths[folds[CFG.FOLD][1]]\n    train_features = np.array(train_features.tolist() + feats1)\n    \n    train_paths = train_features#feat_paths[folds[CFG.FOLD][0]]\n    valid_paths = valid_features#feat_paths[folds[CFG.FOLD][1]]\n    \n    #train_paths = train_paths.append([train_paths[train_paths.fractured==1]] * 1)\n    \n    train_dataset = SpineDataset(train_paths, None, 1)\n    valid_dataset = SpineDataset(valid_paths, None, 0)\n    \n    if CFG.DDP and CFG.DDP_INIT_DONE:\n        train_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_dataset, shuffle=True)\n        train_loader = DataLoader(train_dataset, batch_size=CFG.TRN_BS, sampler=train_sampler, num_workers=8, pin_memory=False)\n        \n        \n        valid_loader = DataLoader(valid_dataset, batch_size=CFG.VAL_BS, shuffle=False, num_workers=8, pin_memory=False)\n        #valid_sampler = torch.utils.data.distributed.DistributedSampler(dataset=valid_dataset, shuffle=False)\n        #valid_loader = DataLoader(valid_dataset, batch_size=CFG.VAL_BS, sampler=valid_sampler, shuffle=False, num_workers=8, pin_memory=False)\n    else:\n        train_loader = DataLoader(train_dataset, batch_size=CFG.TRN_BS, shuffle=True, num_workers=8, pin_memory=False)\n        valid_loader = DataLoader(valid_dataset, batch_size=CFG.VAL_BS, shuffle=False, num_workers=8, pin_memory=False)\n    \n    return train_loader, valid_loader#, train_data, valid_data\n\ntrain_loader, valid_loader = get_loaders()\nfor d in valid_loader: break\nplt.imshow(d[0][0].numpy().transpose(1, 2, 0)[:, :, :3], cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:25:30.042752Z","iopub.execute_input":"2022-10-31T15:25:30.043411Z","iopub.status.idle":"2022-10-31T15:25:31.498616Z","shell.execute_reply.started":"2022-10-31T15:25:30.043368Z","shell.execute_reply":"2022-10-31T15:25:31.497536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)\n    \nclass Model(nn.Module):\n    def __init__(self, seq_dim=CFG.DIM):\n        super(Model, self).__init__()\n        \n        m = 1\n        \n        base = 2048\n        c = 512\n        \n        self.c = c\n        \n        self.lstm1 = nn.GRU(base*m, c*m, bidirectional=True, batch_first=True)\n        self.lstm2 = nn.GRU(c*2*m, c*m, bidirectional=True, batch_first=True)\n        \n        self.attention1 = Attention(c * m * 2, seq_dim)\n        #self.attention2 = UFOAttention(d_model=512*m*2, d_k=512*m*2, d_v=512*m*2, h=8)\n        \n        self.conv1 = nn.Conv1d(seq_dim, 1, 1)\n        \n        self.lstm_bn1 = nn.BatchNorm1d(seq_dim)\n        self.lstm_bn2 = nn.BatchNorm1d(seq_dim)\n        \n        self.att_bn1 = nn.BatchNorm1d(c*2*m)\n        \n        self.conv_bn1 = nn.BatchNorm1d(1)\n        \n        self.clf = nn.Linear(c*2*m*2, 1)\n        \n        self.flatten = nn.Flatten()\n        self.relu = nn.ReLU()\n        self.tanh = nn.Tanh()\n        self.avgpool = nn.AdaptiveAvgPool1d(1)\n        \n        self.dropout = nn.Dropout(0.2)\n        \n        self.final_classifier = nn.Linear(c*2*2*7, 8)\n        \n        self.sigmoid = nn.Sigmoid()\n        \n    def forward(self, inp):\n        x, _ = self.lstm1(inp)\n        x = self.tanh(x)\n        \n        #x = self.dropout(x)\n        \n        x = self.lstm_bn1(x)\n        \n        x, _ = self.lstm2(x)\n        x = self.tanh(x)\n        \n        x = self.lstm_bn2(x)\n        x = self.relu(x)\n        \n        #x = self.dropout(x)\n        \n        x_conv = self.conv1(x)\n        x_conv = x_conv[:, 0]\n        \n        #x, _ = self.lstm2(x)\n        #print(x.shape)\n        \n        #max_pool, _ = torch.max(x, 1)\n        #max_pool = self.avgpool(x.transpose(1, 2))[:, :, 0]\n        #print(max_pool.shape)\n        \n        #x = self.attention2(x, x, x)\n        att_pool = self.attention1(x, mask=None)\n        #print(att_pool.shape)\n        #print(att_pool.shape)\n        \n        x = att_pool#torch.cat([max_pool, att_pool], -1)\n        \n        x = self.att_bn1(x)\n        x = self.relu(x)\n        \n        x = torch.cat([x, x_conv], -1)\n        \n        x = self.dropout(x)\n        \n        #logits = self.clf(x)\n        \n        features = x.reshape(x.shape[0]//7, 7, self.c*2*2)\n        features = nn.Flatten(1, 2)(features)\n        \n        logits = self.final_classifier(features)\n        \n        return logits\n    \ndef define_criterion_optimizer_scheduler_scaler(model):\n    criterion = nn.BCEWithLogitsLoss(pos_weight=torch.as_tensor([1.5,]).cuda()).cuda()\n    optimizer = optim.RAdam(model.parameters(), lr=CFG.LR, weight_decay=1e-5)\n    \n    #optimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-5)\n\n    scheduler = pb.optimizers.lr_scheduler.LinearWarmupCosineAnnealingLR(optimizer, \n                                                                         warmup_epochs=CFG.WARMUP_EPOCHS, \n                                                                         max_epochs=CFG.EPOCHS,\n                                                                         warmup_start_lr=CFG.WARMUP_LR)\n\n    scaler = torch.cuda.amp.GradScaler(enabled=True)\n    \n    return criterion, optimizer, scheduler, scaler","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:25:31.504082Z","iopub.execute_input":"2022-10-31T15:25:31.504479Z","iopub.status.idle":"2022-10-31T15:25:31.541775Z","shell.execute_reply.started":"2022-10-31T15:25:31.50444Z","shell.execute_reply":"2022-10-31T15:25:31.537283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Model()(torch.as_tensor(np.zeros((14, CFG.DIM, 2048))).float())","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:25:32.499045Z","iopub.execute_input":"2022-10-31T15:25:32.499446Z","iopub.status.idle":"2022-10-31T15:25:33.062306Z","shell.execute_reply.started":"2022-10-31T15:25:32.49941Z","shell.execute_reply":"2022-10-31T15:25:33.061243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(model, loader):\n    model.train()\n    \n    running_loss = 0.0\n    tq = tqdm(loader)\n    \n    for i, batch in enumerate(tq):\n        images, targets, ids = batch\n        \n        #targets[targets==1] = 0.99\n        #targets[targets==0] = 0.01\n\n        images = nn.Flatten(0, 1)(images)\n        #targets = nn.Flatten(0, 1)(targets[:, 1:]).unsqueeze(-1)\n        \n        images = images[:, :CFG.DIM]\n        \n        optimizer.zero_grad()\n\n        with torch.cuda.amp.autocast():\n            images, targets = images.float().cuda(), targets.cuda()\n\n            outputs = model(images)\n            #loss = competiton_loss_row_norm(outputs, targets, 0, 1)\n            loss = criterion(outputs, targets)\n        \n        scaler.scale(loss).backward()\n        \n        scaler.step(optimizer)\n        scaler.update()\n        \n        running_loss += loss.item()\n        \n        tq.set_postfix(loss=running_loss/(i+1))\n        \n        if DEBUG and i==10: break\n    \n    torch.save(model.state_dict(), f\"{OUTPUT_FOLDER}/curr_f{CFG.FOLD}.pth\")\n    \ndef test_one_epoch(path, loader):\n    model = Model()\n    st = torch.load(path, map_location=f\"cpu\")\n    model.eval()\n    model.cuda()\n    model.load_state_dict(st)\n    \n    tq = tqdm(loader)\n    \n    running_loss = 0.\n    \n    OUTPUTS = []\n    TARGETS = []\n    IDS = []\n    \n    for i, batch in enumerate(tq):\n        with torch.no_grad():\n            images, targets, ids = batch\n            images, targets = images.float().cuda(), targets.cuda()\n            \n            images = nn.Flatten(0, 1)(images)\n            #targets = nn.Flatten(0, 1)(targets[:, 1:]).unsqueeze(-1)\n\n            images = images[:, :CFG.DIM]\n            \n            outputs = model(images)\n            \n            #loss = criterion(outputs, targets)\n            \n            #outputs = torch.max(outputs, -1)[0].unsqueeze(-1)\n            #targets = torch.max(targets, -1)[0].unsqueeze(-1)\n            \n            OUTPUTS.append(model.sigmoid(outputs).detach().cpu().numpy()[0])\n            TARGETS.append(targets.detach().cpu().numpy()[0])\n            IDS.append(ids)\n            \n        #running_loss += loss.item()\n        \n        #tq.set_postfix(valid_loss=running_loss/(i+1))\n    \n    #return np.array(OUTPUTS), np.array(TARGETS)\n    \n    OUTPUTS = np.stack(OUTPUTS)\n    TARGETS = np.stack(TARGETS)\n    IDS = np.array(IDS)\n    \n    valid_targets = TARGETS\n    \n    loss = competiton_loss_row_norm(torch.as_tensor(OUTPUTS).float().clip(0.01, 0.99), torch.as_tensor(valid_targets).float()).item()\n    auc = roc_auc_score(valid_targets, OUTPUTS)\n    #ens = competiton_loss_row_norm(torch.as_tensor((OUTPUTS+P_)/2).float().clip(0.01, 0.99), torch.as_tensor(valid_targets).float()).item()\n    \n    print(f\"AUC: {auc}\")\n    print(f\"LOSS: {loss}\")\n    #print(f\"ENS: {ens}\")\n    \n    #return min(loss, ens)\n    \n    return loss\n\ndef run(model, train_loader, valid_loader):\n    epochs = []\n    scores = []\n    \n    best_score = float('inf')\n    for epoch in range(CFG.EPOCHS):\n        train_one_epoch(model, train_loader)\n        \n        score = test_one_epoch(f\"{OUTPUT_FOLDER}/curr_f{CFG.FOLD}.pth\", valid_loader)\n\n        print(f\"EPOCH {epoch+1} SCORE:\", score)\n        \n        epochs.append(epoch)\n        scores.append(score)\n\n        pd.DataFrame({'epoch': epochs, 'score': scores}).to_csv(f\"{OUTPUT_FOLDER}/log_f{CFG.FOLD}.csv\", index=False)\n\n        if score<best_score:\n            torch.save(model.state_dict(), f\"{OUTPUT_FOLDER}/best_f{CFG.FOLD}.pth\")\n            best_score = score\n\n        try:\n            command.run(['rm', '-r', CFG.cache_dir])\n            pass\n        except:\n            pass\n\n        os.makedirs(CFG.cache_dir, exist_ok=1)\n\n        scheduler.step()\n        \n        #break","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:26:08.380917Z","iopub.execute_input":"2022-10-31T15:26:08.38132Z","iopub.status.idle":"2022-10-31T15:26:08.400775Z","shell.execute_reply.started":"2022-10-31T15:26:08.38129Z","shell.execute_reply":"2022-10-31T15:26:08.399548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for F in range(5):\n    \n    print(f\"FOLD {F}\")\n    \n    CFG.FOLD = F\n    train_loader, valid_loader = get_loaders()\n    \n    model = Model().cuda()\n    \n    criterion, optimizer, scheduler, scaler = define_criterion_optimizer_scheduler_scaler(model)\n\n    run(model, train_loader, valid_loader)\n    \n    print()\n    print()\n    \n    if DEBUG: break","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:26:14.327833Z","iopub.execute_input":"2022-10-31T15:26:14.328194Z","iopub.status.idle":"2022-10-31T15:26:43.920955Z","shell.execute_reply.started":"2022-10-31T15:26:14.328164Z","shell.execute_reply":"2022-10-31T15:26:43.919283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}