{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install iterative-stratification\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nimport pandas as pd\nmskf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=2020)\n\ndf = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train.csv\")\n\n\n\nfor fold, ( _, val_) in enumerate(mskf.split(X=df, y=df[df.columns[1:]])):\n    df.loc[val_ , \"fold\"] = fold\n    \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-17T22:00:01.662891Z","iopub.execute_input":"2022-09-17T22:00:01.663453Z","iopub.status.idle":"2022-09-17T22:00:14.832437Z","shell.execute_reply.started":"2022-09-17T22:00:01.663381Z","shell.execute_reply":"2022-09-17T22:00:14.831424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys,gc,os,random,time,math\nimport matplotlib.pyplot as plt\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom collections import defaultdict, Counter\nfrom  torch.cuda.amp import autocast, GradScaler \nimport cv2\n\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\nimport sklearn.metrics as metrics\nfrom sklearn.model_selection import StratifiedKFold,GroupKFold\nfrom sklearn.metrics import log_loss\nfrom functools import partial\nfrom tqdm import tqdm\nfrom sklearn.metrics import precision_score,recall_score,f1_score,log_loss\nfrom  sklearn.metrics import accuracy_score as acc\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD,AdamW\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau,CosineAnnealingWarmRestarts\nimport transformers as T\n\n\nfrom torch.optim.swa_utils import AveragedModel, SWALR\n\n###\nimport logging","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:14.834322Z","iopub.execute_input":"2022-09-17T22:00:14.834911Z","iopub.status.idle":"2022-09-17T22:00:16.663195Z","shell.execute_reply.started":"2022-09-17T22:00:14.834879Z","shell.execute_reply":"2022-09-17T22:00:16.662189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def func(study):\n    seq = np.load(f\"../input/stg2get-emb-all-512/3darray/{study}.npy\")\n    return seq.shape[0]\nvfunc =  np.vectorize(func)\n\n#df[\"num_slice\"]=vfunc(df[\"StudyInstanceUID\"].values)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:16.664685Z","iopub.execute_input":"2022-09-17T22:00:16.665311Z","iopub.status.idle":"2022-09-17T22:00:16.671109Z","shell.execute_reply.started":"2022-09-17T22:00:16.665272Z","shell.execute_reply":"2022-09-17T22:00:16.67002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\n#### dataset ==============\nclass TrainDataset(torch.utils.data.Dataset):\n    def __init__(self, df,train=True):\n        self.df = df\n        self.cols = ['C1', 'C2', 'C3', 'C4', 'C5','C6', 'C7',\"patient_overall\"]\n\n        self.crop = 300\n        self.flip = 0.5\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        study = self.df[\"StudyInstanceUID\"].values[idx]\n        #seq = np.load(f\"/home/u094724e/rsna2022/dino/exp001/50epoch/{study}.npy\")#n_seq,2304\n        seq = np.load(f\"../input/1w3s-get-emb-all-debug/3darray/{study}.npy\")\n        seq= seq[:,-768:]\n        mask = torch.ones(seq.shape[0])\n        #seq_len = self.df[\"num_slice\"].values[idx]\n\n        \n        label = self.df[self.cols].to_numpy()[idx]\n        seq = torch.from_numpy(seq).float()\n        label = torch.tensor(label).float()\n\n        return seq,mask,label#,torch.tensor(seq_len)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:16.673928Z","iopub.execute_input":"2022-09-17T22:00:16.674339Z","iopub.status.idle":"2022-09-17T22:00:16.685956Z","shell.execute_reply.started":"2022-09-17T22:00:16.674294Z","shell.execute_reply":"2022-09-17T22:00:16.684929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dataset = TrainDataset(df)\n\n#train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, collate_fn=collate_fn,shuffle=True, num_workers=12,pin_memory=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:16.687422Z","iopub.execute_input":"2022-09-17T22:00:16.687843Z","iopub.status.idle":"2022-09-17T22:00:16.698414Z","shell.execute_reply.started":"2022-09-17T22:00:16.687808Z","shell.execute_reply":"2022-09-17T22:00:16.697294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.utils.rnn as rnn\ndef collate_fn(batch):\n    #seq,label,seq_len= list(zip(*batch))\n    seq,mask,label= list(zip(*batch))\n    seq = rnn.pad_sequence(seq).permute((1,0,2))\n    mask = rnn.pad_sequence(mask).permute((1,0))\n    #seq = torch.stack(seq)\n    label = torch.stack(label)\n    #seq_len = torch.stack(seq_len)\n    return seq,mask,label#,seq_len\n\ndef test_collate_fn(batch):\n    seq = rnn.pad_sequence(batch).permute((1,0,2))\n    #seq = torch.stack(seq)\n    #label = torch.stack(label)\n    #seq_len = torch.stack(seq_len)\n    return seq#, label#,seq_len","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:16.699798Z","iopub.execute_input":"2022-09-17T22:00:16.700275Z","iopub.status.idle":"2022-09-17T22:00:16.70994Z","shell.execute_reply.started":"2022-09-17T22:00:16.700219Z","shell.execute_reply":"2022-09-17T22:00:16.708894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.nn import AdaptiveAvgPool2d, AdaptiveMaxPool2d\n\n\n# From: https://github.com/filipradenovic/cnnimageretrieval-pytorch/blob/master/cirtorch/layers/pooling.py\ndef gem_1d(x, p=3, eps=1e-6):\n    return F.avg_pool1d(x.clamp(min=eps).pow(p), (x.size(-1),)).pow(1./p)\n\n\ndef gem_2d(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\n\ndef gem_3d(x, p=3, eps=1e-6):\n    return F.avg_pool3d(x.clamp(min=eps).pow(p), (x.size(-3), x.size(-2), x.size(-1))).pow(1./p)\n\n\n_GEM_FN = {\n    1: gem_1d, 2: gem_2d, 3: gem_3d\n}\n\n\nclass GeM(nn.Module):\n\n    def __init__(self, p=3, eps=1e-6, dim=2):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n        self.dim = dim\n\n    def forward(self, x):\n        return _GEM_FN[self.dim](x, p=self.p, eps=self.eps)","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:16.712306Z","iopub.execute_input":"2022-09-17T22:00:16.712577Z","iopub.status.idle":"2022-09-17T22:00:16.72451Z","shell.execute_reply.started":"2022-09-17T22:00:16.712552Z","shell.execute_reply":"2022-09-17T22:00:16.723543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Noop(nn.Module):\n    def __init__(self,*args):\n        super(Noop, self).__init__()\n    def forward(self,x):\n        return x\n\nclass NoopAddDim(nn.Module):\n    def __init__(self):\n        super(NoopAddDim, self).__init__()\n    def forward(self,x):\n        return x.unsqueeze(-1)\n\ndef add_to_dim(x,num_dims,dim=0):\n    while len(x.shape)<num_dims:\n        x=x.unsqueeze(dim)\n    return x\nclass DummyEmbd(nn.Module):\n    def __init__(self,out_size,dtype=torch.float32):\n        super(DummyEmbd, self).__init__()\n        self.out_size=out_size\n        self.dtype=dtype\n    def forward(self,x):\n        return torch.zeros(x.shape+(self.out_size,),dtype=self.dtype,device=x.device)\n\nimport math\ndef calc_positional_encoder(d_model, max_seq_len = 32):\n        # create constant 'pe' matrix with values dependant on\n        # pos and i\n        pe = torch.zeros(max_seq_len, d_model)\n        for pos in range(max_seq_len):\n            for i in range(0, d_model, 2):\n                pe[pos, i] = \\\n                math.sin(pos / (10000 ** ((2 * i)/d_model)))\n                pe[pos, i + 1] = \\\n                math.cos(pos / (10000 ** ((2 * (i + 1))/d_model)))\n        return pe/(d_model**0.5)\n\nclass SEModule(nn.Module):\n\n    def __init__(self, channels, reduction):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n        self.fc1 = nn.Conv1d(channels, channels // reduction, kernel_size=1,\n                             padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv1d(channels // reduction, channels, kernel_size=1,\n                             padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x\n\nclass GaussianDropout(nn.Module):\n\n    def __init__(self, p: float):\n        \"\"\"\n        Multiplicative Gaussian Noise dropout with N(1, p/(1-p))\n        It is NOT (1-p)/p like in the paper, because here the\n        noise actually increases with p. (It can create the same\n        noise as the paper, but with reversed p values)\n\n        Source:\n        Dropout: A Simple Way to Prevent Neural Networks from Overfitting\n        https://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf\n\n        :param p: float - determines the the standard deviation of the\n        gaussian noise, where sigma = p/(1-p).\n        \"\"\"\n        super().__init__()\n        assert 0 <= p < 1\n        self.t_mean = torch.ones((0,))\n        self.shape = ()\n        self.p = p\n        self.t_std = self.compute_std()\n\n    def compute_std(self):\n        return self.p / (1 - self.p)\n\n    def forward(self, t_hidden):\n        if self.training and self.p > 0.:\n            if self.t_mean.shape != t_hidden.shape:\n                self.t_mean = torch.ones_like(input=t_hidden\n                                              , dtype=t_hidden.dtype\n                                              , device=t_hidden.device)\n            elif self.t_mean.device != t_hidden.device:\n                self.t_mean = self.t_mean.to(device=t_hidden.device, dtype=t_hidden.dtype)\n\n            t_gaussian_noise = torch.normal(self.t_mean, self.t_std)\n            t_hidden = t_hidden.mul(t_gaussian_noise)\n        return t_hidden\n\nclass TransformerModel(nn.Module):\n    def __init__(self,in_size=3,#d_model\n                 dim_feedforward=16,##hidden\n                 n_heads=4,\n                 n_encoders=4,\n                 num_outputs=8,\n                 use_age=False,\n                 max_site_num=7,\n                 use_sex=False,\n                 use_age_diff=False,\n                 use_position_enc=False,\n                 pool=\"avg\",\n                 embed=True):\n        super(TransformerModel, self).__init__()\n        self.in_size=in_size\n        self.do_embed = embed\n        self.encoder_layer =nn.TransformerEncoderLayer(in_size,#d_model(単語の次元..1280)\n                                                       n_heads,\n                                                       dim_feedforward=dim_feedforward)\n#        self.decoder_layer =nn.TransformerDecoderLayer(in_size, 4, dim_feedforward=in_size)\n        self.encoder=nn.TransformerEncoder(self.encoder_layer, n_encoders)\n#        self.decoder=nn.TransformerDecoder(self.decoder_layer, 2)\n        self.egg_emb=nn.Sequential(nn.Linear(12,16),nn.ReLU(),nn.Linear(16,in_size)) \n        ###meta_feature(csvから取ってくるやつ) ==========================\n        self.sex_embd=nn.Embedding(2,in_size) if  use_sex else DummyEmbd(in_size)\n        self.age_embd=nn.Sequential(NoopAddDim(),nn.Linear(1,16),nn.ReLU(),nn.Linear(16,in_size)) if use_age else DummyEmbd(in_size)\n        self.site_embd=nn.Embedding(max_site_num,in_size) if max_site_num>0 else DummyEmbd(in_size)\n        self.age_diff_embd=nn.Sequential(NoopAddDim(),nn.Linear(1,16),nn.ReLU(),nn.Linear(16,in_size)) if use_age else DummyEmbd(in_size)\n        # ==========================\n        if pool in ('avg','concat','gem','max'):\n            self.pool = GeM(dim=1)#nn.AdaptiveAvgPool1d(1)#\n            if pool == 'concat':\n                self.exam_classifier = nn.Linear(in_size*2, num_outputs)\n            else:\n                self.exam_classifier = nn.Linear(in_size, num_outputs)\n        else:\n            self.pool = None\n            self.exam_classifier = nn.Linear(in_size, num_outputs)\n        \n        #self.image_classifier = nn.Linear(in_size, num_outputs)\n        self.pos_embd=calc_positional_encoder(768,max_seq_len=300) if use_position_enc else None\n        self.layer1 = nn.Sequential(\n                nn.Conv1d(in_size, in_size, kernel_size=13, stride=1, padding=0, bias=False),\n                nn.BatchNorm1d(in_size),\n                #nn.ReLU(inplace=True),\n                nn.Mish(),\n                nn.Conv1d(in_size,in_size*2, kernel_size=7, stride=1, padding=0, bias=False),\n                nn.BatchNorm1d(in_size*2),\n                nn.Mish(),\n                #SEModule(in_size*2, in_size//2),#だめ\n                nn.Conv1d(in_size*2,in_size, kernel_size=7, stride=2, padding=0, bias=False),\n                nn.BatchNorm1d(in_size),\n                nn.Mish(),\n                #nn.Dropout(),#AUCはよい bceはだめ\n                #SEModule(dim, dim//4),\n        )\n        self.gru = nn.GRU(in_size, in_size//2, bidirectional=True, batch_first=True, num_layers=2)\n        self.gru_afetr = nn.GRU(in_size, in_size, bidirectional=False, batch_first=True, num_layers=2)\n        \n        self.dropout = nn.Dropout(0.2)\n        self.Gdropout = GaussianDropout(0.5)\n    #@time_function_execution\n    def forward(self, x,sex=None,age=None,site=None,age_diff=None,mask=None):\n        if self.pos_embd is not None:\n            if self.pos_embd.device!=x.device:\n                self.pos_embd = self.pos_embd.to(x.device)\n        ###meta_feature(csvから取ってくるやつ) ==========================...yuvalのsiimより??\n        #x = x if sex is None else x + self.sex_embd(sex)\n        #x = x if age is None else x + self.age_embd(age)\n        #x = x if site is None else x + self.site_embd(site)\n        #x = x if age_diff is None else x + self.age_diff_embd(age_diff)\n        # ==========================\n        #print(self.pos_embd[:x.shape[1]][None].shape,x.shape)\n        x = x if self.pos_embd is None else x + self.pos_embd[:x.shape[1]][None]\n        x = x if mask is None else x*mask.unsqueeze(-1)\n        #x = x if self.do_embed ==False else self.layer1(x.permute(0, 2, 1)).permute(0, 2, 1)#self.egg_emb(x)\n        x = x if self.do_embed ==False else self.gru(x)[0]#self.egg_emb(x)\n        #before 1dcnnの方がやはり良さそうだが、overfitしがち...1dcnn部分を薄くするかどうか..\n\n        x = self.encoder(x) #,mask=s_mask #bs,len,depth\n        #x = self.gru_afetr(x)[0]\n        #x = torch.stack([self.dropout(x) for _ in range(5)], 0).mean(0)\n        #x = self.Gdropout(x)#5e-3改善　どっちでも良さそう\n        if self.pool:\n            pooled = self.pool(x.permute(0,2,1))\n            #pooled = torch.stack([self.dropout(pooled) for _ in range(5)], 0).mean(0)\n            feature = pooled[:,:,0]\n            \n            exam = self.exam_classifier(feature)\n        else:\n            feature = x[:,0,:]\n            exam = self.exam_classifier(feature)##0番目の\"単語\"にクラス分類のための特徴が集約されるように学習\n        return exam\n\n\n\nclass gru_model(nn.Module):\n    def __init__(self,input_ch,hidden_size=256,pool=\"avg\"):\n        super().__init__()\n        \n        self.gru = nn.GRU(input_ch, hidden_size, bidirectional=True, batch_first=True, num_layers=2)\n        self.gru2 = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True, num_layers=2)\n        \n        #if pool in ('avg','concat','gem','max'):\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        if pool == 'concat': hidden_size *= 2\n\n        self.exam_predictor = nn.Linear(hidden_size*2, 8)\n        \n    #@time_function_execution\n    def forward(self, embeds):\n        #embeds = embeds.permute(0, 2, 1)\n        embeds, _ = self.gru(embeds)\n        embeds, _ =self.gru2(embeds)\n        embeds = self.pool(embeds.permute(0,2,1))[:,:,0]\n        \n        exam_pred = self.exam_predictor(embeds)\n        return exam_pred\n\n\n#### model ================\n\n\n###  loss =============\n\nclass competiton_loss_row_norm(nn.Module):\n    \n    # https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/discussion/340392\n    def __init__(self,device):\n        super().__init__()\n        eacl_w = torch.full([7],2)\n        self.loss_fn = nn.BCEWithLogitsLoss(reduction=\"none\") \n        self.competition_weights = {\n    '-' : torch.tensor([1, 1, 1, 1, 1, 1, 1,7], dtype=torch.float, device=device),\n    '+' : torch.tensor([2, 2, 2, 2, 2, 2, 2,14], dtype=torch.float, device=device),\n}\n        \n        \n\n    def forward(self, y_hat, y):\n        loss = self.loss_fn(y_hat, y)\n        weights = y * self.competition_weights['+'] + (1 - y) * self.competition_weights['-']\n        loss = (loss * weights).sum(axis=1)\n        w_sum = weights.sum(axis=1)\n        loss = torch.div(loss, w_sum)\n        return loss.mean()","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:16.727659Z","iopub.execute_input":"2022-09-17T22:00:16.728056Z","iopub.status.idle":"2022-09-17T22:00:16.772944Z","shell.execute_reply.started":"2022-09-17T22:00:16.728026Z","shell.execute_reply":"2022-09-17T22:00:16.770657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers.models.distilbert.modeling_distilbert import Transformer as _Transformer\n\nclass Config:\n\n    def __init__(self, **kwargs):\n        for k,v in kwargs.items():\n            setattr(self,k,v)\n\n\nclass Transformer(nn.Module):\n\n    def __init__(self,\n                 num_classes,\n                 embedding_dim=768,\n                 hidden_dim=768,\n                 n_layers=4,\n                 n_heads=16,\n                 dropout=0.2,\n                 attn_dropout=0.1,\n                 seq_len=256,\n                 output_attns=False,\n                 act_fn='gelu',\n                 chunk=False,\n                 reverse=False):\n        super().__init__()\n        config = Config(**{\n                'dim': embedding_dim,\n                'hidden_dim': hidden_dim,\n                'n_layers': n_layers,\n                'n_heads': n_heads,\n                'dropout': dropout,\n                'attention_dropout': attn_dropout,\n                'output_attentions': output_attns,\n                'activation': act_fn,\n                'output_hidden_states': False,\n                'chunk_size_feed_forward': 0\n            })\n\n        self.transformer = _Transformer(config)\n        self.classifier = nn.Linear(embedding_dim, num_classes)\n        self.chunk = chunk\n        self.reverse = reverse\n\n    def classify(self, x):\n        out = self.classifier(x)\n        if self.classifier.out_features == 1:\n            return out[...,0]\n        else:\n            return out\n\n    def forward_tr(self, x, mask):\n        output = self.transformer(x, attn_mask=mask, head_mask=[None]*x.size(1))\n        return self.classify(output[0])\n\n    def forward(self, x):\n        x, mask = x\n        if self.chunk and not self.training:\n            chunk_size = 64\n            # Create placeholder for predictions\n            output = torch.zeros((x.size(0), x.size(1))).float().to(x.device)\n            chunks = torch.zeros_like(output).to(x.device)\n            indices = torch.arange(0, x.size(1), chunk_size//2)\n            if x.size(1) - indices[-1] < 16:\n                indices = indices[:-1]\n            for ind, i in enumerate(indices):\n                if ind == len(indices)-1:\n                    tmpx, tmpmask = x[:,i:], mask[:,i:]\n                else:\n                    tmpx, tmpmask = x[:,i:i+chunk_size], mask[:,i:i+chunk_size]\n                chunkout = self.forward_tr(tmpx, tmpmask)\n                output[:,i:i+chunkout.size(1)] += chunkout\n                chunks[:,i:i+chunkout.size(1)] += 1.0\n            output /= chunks\n            return output\n        else:\n            if not self.training and self.reverse:\n                xrev = torch.flip(x, dims=(1,))\n                mrev = torch.flip(mask, dims=(1,))\n                reverseout = self.forward_tr(xrev, mrev)\n                out = self.forward_tr(x, mask)\n                return torch.mean(torch.stack([out, torch.flip(reverseout, dims=(1,))]), dim=0)\n            return self.forward_tr(x, mask)\n\n\n\n\nSEQ_POOLING = {\n    'gem': GeM(dim=1),\n    'avg': nn.AdaptiveAvgPool1d(1),\n    'max': nn.AdaptiveMaxPool1d(1)\n}\nclass TransformerCls(nn.Module):\n\n    def __init__(self,\n                 embedding_dim=768,\n                 hidden_dim=768,\n                 n_layers=4,\n                 n_heads=16,\n                 dropout=0.2,\n                 attn_dropout=0.1,\n                 output_attns=False,\n                 act_fn='gelu',\n                 pool=None):\n        super().__init__()\n        config = Config(**{\n                'dim': embedding_dim,\n                'hidden_dim': hidden_dim,\n                'n_layers': n_layers,\n                'n_heads': n_heads,\n                'dropout': dropout,\n                'attention_dropout': attn_dropout,\n                'output_attentions': output_attns,\n                'activation': act_fn,\n                'output_hidden_states': False,\n                'chunk_size_feed_forward': 0\n            })\n\n        self.transformer = _Transformer(config)\n        if pool in ('avg','concat','gem','max'):\n            self.pool = SEQ_POOLING[pool]\n            if pool == 'concat': embedding_dim *= 2\n        else:\n            self.pool = None\n\n        self.classifier = nn.Linear(embedding_dim, 8)\n\n    def forward(self,x, mask):\n        output = self.transformer(x, attn_mask=mask, head_mask=[None]*x.size(1))\n        if self.pool:\n            output = self.pool(output[0].transpose(-1,-2))[:,:,0]\n        else:\n            output = output[0][:,0]\n        output = self.classifier(output)\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:16.779232Z","iopub.execute_input":"2022-09-17T22:00:16.781955Z","iopub.status.idle":"2022-09-17T22:00:18.31628Z","shell.execute_reply.started":"2022-09-17T22:00:16.781907Z","shell.execute_reply":"2022-09-17T22:00:18.314828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = T.AutoConfig.from_pretrained(\"microsoft/deberta-base\")\nconfig\n\n#print(model)","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:18.32165Z","iopub.execute_input":"2022-09-17T22:00:18.321954Z","iopub.status.idle":"2022-09-17T22:00:19.046317Z","shell.execute_reply.started":"2022-09-17T22:00:18.32192Z","shell.execute_reply":"2022-09-17T22:00:19.04536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class transformer_HF(nn.Module):\n    \"\"\"\n    huggingfaceのいろんなモデルのencoderだけ使う\n    \"\"\"\n    def __init__(self,model_name=\"microsoft/deberta-v3-small\",in_size=768,num_hidden_layers=2,pool=\"gem\"):\n        super(transformer_HF, self).__init__()\n        config = T.AutoConfig.from_pretrained(model_name)\n        config.num_hidden_layers=num_hidden_layers\n        config.hidden_size=in_size\n        config.num_attention_heads=4\n        config.intermediate_size = 256\n        self.config = config\n        if \"deberta-v3\" in model_name:\n            self.model = T.models.deberta_v2.modeling_deberta_v2.DebertaV2Encoder(config)\n        else:\n            self.model = T.models.deberta.modeling_deberta.DebertaEncoder(config)\n        self.gru = nn.GRU(in_size, in_size//2, bidirectional=True, batch_first=True, num_layers=2)\n        \n        if pool in ('avg','concat','gem','max'):\n            self.pool = GeM(dim=1)#nn.AdaptiveAvgPool1d(1)#\n            if pool == 'concat':\n                self.exam_classifier = nn.Linear(in_size*2, 8)\n            else:\n                self.exam_classifier = nn.Linear(in_size, 8)\n    def forward(self,embedding_output,mask):\n        embedding_output = self.gru(embedding_output)[0]\n        mask = torch.ones(embedding_output.shape[0],embedding_output.shape[1]).to(embedding_output.device)\n        x = self.model(embedding_output,mask)[\"last_hidden_state\"]\n        pooled = self.pool(x.permute(0,2,1))\n            #pooled = torch.stack([self.dropout(pooled) for _ in range(5)], 0).mean(0)\n        feature = pooled[:,:,0]\n            \n        exam = self.exam_classifier(feature)\n        return exam","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:19.047843Z","iopub.execute_input":"2022-09-17T22:00:19.048883Z","iopub.status.idle":"2022-09-17T22:00:19.063435Z","shell.execute_reply.started":"2022-09-17T22:00:19.048822Z","shell.execute_reply":"2022-09-17T22:00:19.0623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = TransformerModel(in_size=768,\n    dim_feedforward=256,#256 is ok\n    num_outputs=8,n_heads=4,\n    n_encoders=4,#4 is OK\n    pool =\"avg\",\n    #use_position_enc=True,\n    embed=True)\n#model","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:19.065128Z","iopub.execute_input":"2022-09-17T22:00:19.065807Z","iopub.status.idle":"2022-09-17T22:00:19.498299Z","shell.execute_reply.started":"2022-09-17T22:00:19.065771Z","shell.execute_reply":"2022-09-17T22:00:19.497243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs =7\nbatch_size = 32\nwarmup = 1\nlr = 1e-4\namp=True","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:19.499879Z","iopub.execute_input":"2022-09-17T22:00:19.500282Z","iopub.status.idle":"2022-09-17T22:00:19.50554Z","shell.execute_reply.started":"2022-09-17T22:00:19.500243Z","shell.execute_reply":"2022-09-17T22:00:19.504239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef train_fn(fold,folds):\n\n    cols = ['patient_overall', 'C1', 'C2', 'C3', 'C4', 'C5','C6', 'C7']\n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    print(f\"### fold: {fold} ###\")\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n\n    val_folds = folds.loc[val_idx].reset_index(drop=True)\n    tra_folds = folds.loc[trn_idx].reset_index(drop=True)\n\n   \n    train_dataset = TrainDataset(tra_folds)\n    valid_dataset = TrainDataset(val_folds)\n\n    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, collate_fn=collate_fn,shuffle=True, num_workers=12,pin_memory=True)\n    valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, collate_fn=collate_fn, shuffle=False, num_workers=12,pin_memory=True)\n\n\n    ###  model select ============\n\n    #model = gru_model(2304).to(device)\n    model = TransformerModel(in_size=768,\n    dim_feedforward=256,#256 is ok\n    num_outputs=8,n_heads=4,\n    n_encoders=4,#4 is OK\n    pool =\"avg\",\n    #use_position_enc=True,\n    embed=True)#.to(device)\n\n    model = transformer_HF(\"microsoft/deberta-base\").to(device)\n    #model =TransformerCls().to(device)\n    # ============\n\n\n    ###  optim select ============\n\n    \n    optimizer = AdamW(model.parameters(), lr=lr,weight_decay=5e-5)\n    # ============\n\n    ###  scheduler select ============\n    #if CFG.train.scheduler.name==\"cosine\":\n    #    scheduler = CosineAnnealingLR(optimizer, T_max=CFG.train.epochs, eta_min=CFG.train.scheduler.min_lr)\n    #elif CFG.train.scheduler.name==\"cosine_warmup\":\n    scheduler =T.get_cosine_schedule_with_warmup(optimizer,\n        num_warmup_steps=len(train_loader)*warmup,\n        num_training_steps=len(train_loader)*epochs)\n\n    # ============\n\n    ###  loss select ============\n    criterion=competiton_loss_row_norm(device)\n    val_criterion = competiton_loss_row_norm(torch.device(\"cpu\"))\n\n\n    print(criterion)\n    ###  loss select ============\n\n    sigmoid = nn.Sigmoid()\n    scaler = torch.cuda.amp.GradScaler()\n    best_score = 0\n    best_loss = np.inf\n    best_preds = None\n        \n    for epoch in range(epochs):\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.\n        train_loss = []\n\n        tk0 = tqdm(enumerate(train_loader), total=len(train_loader))\n\n        for i, (images,masks, labels) in tk0:\n            optimizer.zero_grad()\n            images = images.to(device).float()\n            masks = masks.to(device)\n            labels = labels.to(device).float()\n            \n            rand = np.random.rand()\n            rand = 1\n            #if epoch>6:rand=1 #ないほうがよい\n            #if 0.5>rand:images, y_a, y_b, lam = mixup_data(images, labels)\n\n            with autocast(enabled=amp):\n\n                y_preds = model(images,mask=masks)\n                #if 0.5>rand:loss = mixup_criterion(criterion, y_preds, y_a, y_b, lam)\n                loss = criterion(y_preds,labels)\n\n                \n                #loss = competiton_loss(y_preds,labels,device)\n\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n                scheduler.step()\n                        \n\n            avg_loss += loss.item() / len(train_loader)\n            loss_np = loss.detach().cpu().numpy()\n            train_loss.append(loss_np)\n        avg_loss = np.mean(train_loss)\n\n\n        model.eval()\n        avg_val_loss = 0.\n        LOGITS = []\n        TARGETS = []\n        preds = []\n        valid_labels = []\n        tk1 = tqdm(enumerate(valid_loader), total=len(valid_loader))\n\n        for i, (images,masks,labels) in tk1:\n            images = images.to(device).float()\n            labels = labels.to(device).float()\n            masks = masks.to(device)\n            with torch.no_grad():\n                logits = model(images,mask=masks)\n                LOGITS.append(logits.detach().cpu())\n                TARGETS.append(labels.detach().cpu())\n\n        \n            #valid_labels.append(labels.to('cpu').numpy())\n            #preds.append(y_preds.to('cpu').numpy())\n            #avg_val_loss += loss.item() / len(valid_loader)\n        #preds = np.concatenate(preds)\n        #valid_labels = np.concatenate(valid_labels)\n            \n        avg_val_loss = val_criterion(torch.cat(LOGITS), torch.cat(TARGETS)).numpy()\n        preds = torch.sigmoid(torch.cat(LOGITS)).numpy().squeeze()    \n        valid_labels = torch.cat(TARGETS).numpy()\n        \n        mean_pred = torch.stack([torch.from_numpy(valid_labels.mean(axis=0))]*valid_labels.shape[0]).logit()\n        #mean_pred = torch.full_like(torch.cat(TARGETS),0)\n        mean_loss = val_criterion(mean_pred, torch.cat(TARGETS)).numpy()\n        print(\"MEAN\",mean_loss)\n\n\n\n        print(preds.shape,valid_labels.shape)\n        print(valid_labels.mean(axis=0))\n\n        elapsed = time.time() - start_time\n\n\n        print(f'  Epoch {epoch+1} - avg_train_loss: {avg_loss:.6f}  avg_val_loss: {avg_val_loss:.6f}  time: {elapsed:.0f}s')\n\n        if best_loss>avg_val_loss:#pr_auc best\n            best_loss = avg_val_loss\n            best_preds = preds\n            print(f'  Epoch {epoch+1} - Save Best loss: {best_loss:.4f}')\n            torch.save(model.state_dict(), f'fold{fold}_002_best_loss.pth')\n\n        for i in range(8):\n            col = f\"pred_{i}\"\n            val_folds[col]=best_preds[:,i]\n\n    return best_preds, valid_labels,val_folds\n","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:19.507745Z","iopub.execute_input":"2022-09-17T22:00:19.508612Z","iopub.status.idle":"2022-09-17T22:00:19.533756Z","shell.execute_reply.started":"2022-09-17T22:00:19.508573Z","shell.execute_reply":"2022-09-17T22:00:19.532609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nvalid_labels = []\noof = pd.DataFrame()\nfor fold in range(5):\n    seed_torch(seed=42)\n    _preds, _valid_labels,_oof_val = train_fn(fold,df)\n    preds.append(_preds)\n    valid_labels.append(_valid_labels)\n    oof = pd.concat([oof,_oof_val])\npreds = np.concatenate(preds)\nvalid_labels = np.concatenate(valid_labels)\n\n\nval_criterion = competiton_loss_row_norm(torch.device(\"cpu\"))\nval_loss = val_criterion(torch.from_numpy(preds).logit(), torch.from_numpy(valid_labels)).numpy()\n\nprint(val_loss)","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:00:19.534807Z","iopub.execute_input":"2022-09-17T22:00:19.535089Z","iopub.status.idle":"2022-09-17T22:25:00.542328Z","shell.execute_reply.started":"2022-09-17T22:00:19.535063Z","shell.execute_reply":"2022-09-17T22:25:00.541052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"dim:256 0.47136664<br>\n→1dcnn:0.47522324<br>\n→msd :0.47203583<br>\n→add abs pos :0.47136256<br>\n→before GRUを2→4 layer :0.47333568<br>\n→before GRUを2→1 layer :0.47549278<br>\n→Gem :0.46634147\n\nref)nn.transformer 0.460\ndeebrta-v1 layer2 w/o GRU 0.4658<br>\ndeebrta-v1 layer2 w/ GRU 0.45925397<br>\n","metadata":{}},{"cell_type":"code","source":"oof.to_csv(f\"oof_002_2.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:25:00.544656Z","iopub.execute_input":"2022-09-17T22:25:00.545376Z","iopub.status.idle":"2022-09-17T22:25:00.579155Z","shell.execute_reply.started":"2022-09-17T22:25:00.545326Z","shell.execute_reply":"2022-09-17T22:25:00.578299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(torch.utils.data.Dataset):\n    def __init__(self, df,train=True):\n        self.df = df\n        self.cols = ['C1', 'C2', 'C3', 'C4', 'C5','C6', 'C7','patient_overall']\n\n        self.crop = 300\n        self.flip = 0.5\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        study = self.df[idx]\n\n        seq = np.load(f\"../input/1w3s-get-emb-all-debug/3darray/{study}.npy\")#n_seq,2304            \n        seq= seq[:,-768:]\n\n        \n        #label = self.df[self.cols].to_numpy()[idx]\n        seq = torch.from_numpy(seq).float()\n        #label = torch.tensor(label).float()\n\n        return seq#,label\ndef inference(model, test_loader,device):\n    model.eval()\n    probs = []\n    LOGITS = []\n\n    for i, images in tqdm(enumerate(test_loader), total=len(test_loader)):\n        images = images.to(device)\n        #print(images.shape)\n        with torch.no_grad():\n            logits = model(images.float(),mask=None)\n            LOGITS.append(logits.detach().cpu())\n\n    PROBS = torch.sigmoid(torch.cat(LOGITS)).numpy().squeeze() \n    return PROBS\ndef get_oof():\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    \n    probs = []\n    all_study = []\n    for fold in range(5):\n        seed_torch(seed=42)\n        test_dataset = TestDataset(oof[oof[\"fold\"]==fold][\"StudyInstanceUID\"].unique())\n        test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32*2,collate_fn=test_collate_fn, shuffle=False, num_workers=2,pin_memory=True)\n\n        weight_path = f\"fold{fold}_002_best_loss.pth\"\n        model = transformer_HF(\"microsoft/deberta-base\").to(device)\n        #model = TransformerModel(in_size=768,dim_feedforward=256,num_outputs=8,n_heads=4,n_encoders=4,pool =\"avg\",embed=True).to(device)\n        state_dict = torch.load(weight_path,map_location=device)\n        model.load_state_dict(state_dict)\n        preds = inference(model, test_loader,device)\n        probs.append(preds)\n        all_study.append(oof[oof[\"fold\"]==fold][\"StudyInstanceUID\"].unique())\n\n    preds = np.concatenate(probs, axis=0)\n    all_study = np.concatenate(all_study, axis=0)\n    return preds,all_study\n    \npreds,all_study = get_oof()\ncols = ['C1', 'C2', 'C3', 'C4', 'C5','C6', 'C7',\"patient_overall\"]\npred_df = dict(zip(all_study,preds))\ndef func_pred(study):\n    return pred_df[study]\n","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:25:00.580603Z","iopub.execute_input":"2022-09-17T22:25:00.580938Z","iopub.status.idle":"2022-09-17T22:25:22.356848Z","shell.execute_reply.started":"2022-09-17T22:25:00.580905Z","shell.execute_reply":"2022-09-17T22:25:22.355056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_cols = [f\"pred_{i}\" for i in range(8)]\npreds_oof = oof[pred_cols].to_numpy()\nval_loss = val_criterion(torch.from_numpy(preds_oof).logit(),torch.from_numpy(oof[cols].to_numpy()).float())\nprint(val_loss)","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:25:22.358798Z","iopub.execute_input":"2022-09-17T22:25:22.359219Z","iopub.status.idle":"2022-09-17T22:25:22.371671Z","shell.execute_reply.started":"2022-09-17T22:25:22.35917Z","shell.execute_reply":"2022-09-17T22:25:22.370511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_array = np.stack([func_pred(study) for study in oof[\"StudyInstanceUID\"].unique()])\nlabels_array= np.stack([oof[oof[\"StudyInstanceUID\"]==study][cols].to_numpy().squeeze() for study in oof[\"StudyInstanceUID\"].unique()]) \nval_loss = val_criterion(torch.from_numpy(predict_array).logit(),torch.from_numpy(labels_array).float())\nprint(val_loss)","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:25:22.373229Z","iopub.execute_input":"2022-09-17T22:25:22.373738Z","iopub.status.idle":"2022-09-17T22:25:24.69522Z","shell.execute_reply.started":"2022-09-17T22:25:22.373701Z","shell.execute_reply":"2022-09-17T22:25:24.694024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nmodel = transformer_HF(\"microsoft/deberta-base\")\nconfig = model.config\nwith open('debrta_config.pickle', 'wb') as f:\n    pickle.dump(config, f)","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:25:24.69663Z","iopub.execute_input":"2022-09-17T22:25:24.697026Z","iopub.status.idle":"2022-09-17T22:25:25.340377Z","shell.execute_reply.started":"2022-09-17T22:25:24.696988Z","shell.execute_reply":"2022-09-17T22:25:25.339096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config","metadata":{"execution":{"iopub.status.busy":"2022-09-17T22:25:25.34204Z","iopub.execute_input":"2022-09-17T22:25:25.34242Z","iopub.status.idle":"2022-09-17T22:25:25.359082Z","shell.execute_reply.started":"2022-09-17T22:25:25.342383Z","shell.execute_reply":"2022-09-17T22:25:25.35684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}