{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nsys.path = [\n    '/kaggle/input/abdominal-test',\n    '../input/timm20221011/pytorch-image-models-master',\n    '../input/smp20210127/segmentation_models.pytorch-master/segmentation_models.pytorch-master',\n    '../input/smp20210127/pretrained-models.pytorch-master/pretrained-models.pytorch-master',\n    '../input/smp20210127/EfficientNet-PyTorch-master/EfficientNet-PyTorch-master',\n    '/kaggle/input/abdominal-utils'\n] + sys.path ","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:23:36.245851Z","iopub.execute_input":"2023-10-21T17:23:36.246188Z","iopub.status.idle":"2023-10-21T17:23:36.257092Z","shell.execute_reply.started":"2023-10-21T17:23:36.246161Z","shell.execute_reply":"2023-10-21T17:23:36.256188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip -q install /kaggle/input/rsna2022whl/rsna-2022-whl/pydicom-2.3.0-py3-none-any.whl\n!pip -q install ../input/pylibjpeg140py3/pylibjpeg-1.4.0-py3-none-any.whl\n!pip install -q /kaggle/input/gdcm-0310/python_gdcm-3.0.22-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n\n!pip install -q /kaggle/input/dicomsdl--0-109-2/dicomsdl-0.109.2-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n\n!cp -r ../input/timm-20220211/pytorch-image-models-master/timm ./timm4smp","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:23:36.258849Z","iopub.execute_input":"2023-10-21T17:23:36.259251Z","iopub.status.idle":"2023-10-21T17:25:13.405781Z","shell.execute_reply.started":"2023-10-21T17:23:36.259219Z","shell.execute_reply":"2023-10-21T17:25:13.404712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport ast\nimport cv2\nimport time\nimport timm\nimport timm4smp\nimport pickle\nimport random\nimport pydicom\nimport dicomsdl\nimport argparse\nimport warnings\nwarnings.filterwarnings('ignore')\nimport threading\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom glob import glob\nimport albumentations\nimport matplotlib.pyplot as plt\nimport segmentation_models_pytorch as smp\nfrom segmentation_models_pytorch.unet.decoder import UnetDecoder\nfrom segmentation_models_pytorch.base import SegmentationHead\n\nimport albumentations\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.cuda.amp as amp\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\n\nimport gdcm\nimport zipfile\nfrom joblib import Parallel, delayed\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom pylab import rcParams\n\ndevice = torch.device('cuda')\ntorch.backends.cudnn.benchmark = True\n\ndevice = 'cuda:0' if torch.cuda.is_available() else 'cpu'\nprint(device)\n\ntimm.__version__, timm4smp.__version__, pydicom.__version__","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-21T17:25:13.407158Z","iopub.execute_input":"2023-10-21T17:25:13.407449Z","iopub.status.idle":"2023-10-21T17:25:22.585412Z","shell.execute_reply.started":"2023-10-21T17:25:13.407425Z","shell.execute_reply":"2023-10-21T17:25:22.584506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\n\ndata_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection'\n\nif DEBUG:\n    data_type = 'train'\nelse:\n    data_type = 'test'\n    \ntest_dir = os.path.join(data_dir, f'{data_type}_images')\n\ndf = pd.read_csv(os.path.join(data_dir, f'{data_type}_series_meta.csv'))\ndf['psid'] = df['patient_id'].astype(str) + '_' + df['series_id'].astype(str)\ndf['image_folder'] = test_dir + '/' + df['patient_id'].astype(str) + '/' + df['series_id'].astype(str)\n\nif DEBUG:\n    df = df.head(10).reset_index(drop=True)\n    # df = df.head(1500).reset_index(drop=True)\n\n\nsub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:22.587509Z","iopub.execute_input":"2023-10-21T17:25:22.587937Z","iopub.status.idle":"2023-10-21T17:25:22.620983Z","shell.execute_reply.started":"2023-10-21T17:25:22.587912Z","shell.execute_reply":"2023-10-21T17:25:22.620109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    image_size_seg = (128, 128, 128)\n    msk_size = image_size_seg[0]\n    n_ch = 5 # for segmentation linespace\n    batch_size_seg = 1\n    num_workers = 2\n\nclass SegConfig1(Config):\n    kernel_type = 'test'\n    backbone = 'resnet18d'\n    model_dir_seg = '/kaggle/input/abdominal-test/segmentations'\n    \n    \nclass OrganBowelConfig1(Config):\n    image_size_cls = 224\n    image_size_organ = 224 # output size after segmentation (for organ and bowel)\n    image_size_bowel = 224\n    n_slice_per_c = 15\n    mul = 2\n    in_chans = 6\n    out_dim = 3\n    \n    slice_type = 1\n    folds = 5\n    \n    organ_kernel_type = 'organ'\n    organ_model_dir_cls = '/kaggle/input/abdominal-test/stage2-organ-type1/save/sz_224/min0_e20_lr1e4'\n    organ_backbone = 'tf_efficientnetv2_s_in21ft1k'\n    \n    bowel_kernel_type = 'bowel'\n    bowel_model_dir_cls = '/kaggle/input/abdominal-test/stage2-bowel-type1/ver1(clean data)'\n    bowel_backbone = 'tf_efficientnetv2_s_in21ft1k'\n    \n    \nclass OrganBowelConfig2(Config):\n    image_size_cls = 224\n    image_size_organ = 224 # output size after segmentation (for organ and bowel)\n    image_size_bowel = 224\n    n_slice_per_c = 15\n    mul = 2\n    in_chans = 6\n    out_dim = 3\n    \n    slice_type = 1\n    folds = 5\n    \n    organ_kernel_type = 'organ'\n    organ_model_dir_cls = '/kaggle/input/abdominal-organ-seres-1ch/organ/seres50_lr1e4_e20_ll'\n    organ_backbone = 'seresnext50_32x4d'\n    \n    bowel_kernel_type = 'bowel'\n    bowel_model_dir_cls = '/kaggle/input/abdominal-organ-seres-1ch/bowel/seres1e4-ce5'\n    bowel_backbone = 'seresnext50_32x4d'\n    \n    \nclass ExtraConfig1(Config):\n    \"\"\"\n        # segmentation head (fix label)\n        feature extractor : sliding_5_384_e3_seres50_seghead\n        sequence model    : s5_m192_sz384_gru_nodropout\n    \"\"\"\n    image_size_extra = 384\n    in_chans = 5 # slide window\n    m_size = 192\n    emb_dim = 2048\n    lstm_size = 512\n    extra_bs = 8\n    \n    forward_type = 1\n    \n    decoder_channels = (256, 128, 64, 32, 16)\n    seg_head_input = 32\n    \n    feature_model_dir = '/kaggle/input/abdominal-extra-seg-gru/feature'\n    feature_kernel_type = 'seg_head'\n    feature_backbone = 'seresnext50_32x4d'\n    \n    sequence_model_dir = '/kaggle/input/abdominal-extra-seg-gru/sequence'\n    sequence_kernel_type = 'seres_384_seghead_m192_gru_nodropout'\n    \n    \nclass ExtraConfig2(Config):\n    \"\"\"\n        feature extractor : sliding_5_384_e3_effv2s\n        sequence model    : s5_m192_sz384_gru_nodropout\n    \"\"\"\n    image_size_extra = 384\n    in_chans = 5 # slide window\n    m_size = 192\n    emb_dim = 1280\n    lstm_size = 512\n    extra_bs = 8\n    \n    forward_type = 2\n    \n    decoder_channels = (256, 160, 64, 48, 24)\n    seg_head_input = 48\n    \n    feature_model_dir = '/kaggle/input/abdominal-extra-effv2s-seg-gru/feature'\n    feature_kernel_type = 'effv2s_e3'\n    feature_backbone = 'tf_efficientnetv2_s_in21ft1k'\n    \n    sequence_model_dir = '/kaggle/input/abdominal-extra-effv2s-seg-gru/sequence'\n    sequence_kernel_type = 'effv2s_384_seghead_m192_gru_nodropout'","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:22.622312Z","iopub.execute_input":"2023-10-21T17:25:22.622583Z","iopub.status.idle":"2023-10-21T17:25:22.634963Z","shell.execute_reply.started":"2023-10-21T17:25:22.622559Z","shell.execute_reply":"2023-10-21T17:25:22.634063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG = Config()\nSegCFG1 = SegConfig1()\nOBCFG1 = OrganBowelConfig1()\nOBCFG2 = OrganBowelConfig2()\nExtraCFG1 = ExtraConfig1()\n# ExtraCFG2 = ExtraConfig2()\n\nseg_configs = [SegCFG1]\n\ndisplay(df)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:22.635912Z","iopub.execute_input":"2023-10-21T17:25:22.636149Z","iopub.status.idle":"2023-10-21T17:25:22.66088Z","shell.execute_reply.started":"2023-10-21T17:25:22.636129Z","shell.execute_reply":"2023-10-21T17:25:22.65988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not DEBUG:\n    if len(glob(os.path.join(test_dir, f\"48843/62825/*\"))) == 1:\n        fast_sub = True\n    else:\n        fast_sub = False\n    print(fast_sub)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:22.661979Z","iopub.execute_input":"2023-10-21T17:25:22.662304Z","iopub.status.idle":"2023-10-21T17:25:22.672661Z","shell.execute_reply.started":"2023-10-21T17:25:22.662275Z","shell.execute_reply":"2023-10-21T17:25:22.671843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicomsdl_to_numpy_image(ds, index=0):\n    info = ds.getPixelDataInfo()\n    if info['SamplesPerPixel'] != 1:\n        raise RuntimeError('SamplesPerPixel != 1')  # number of separate planes in this image\n    shape = [info['Rows'], info['Cols']]\n    dtype = info['dtype']\n    outarr = np.empty(shape, dtype=dtype)\n    ds.copyFrameData(index, outarr)\n    return outarr\n\n\ndef dicomsdl_read_one(path, img_type):\n    if '/'.join(path.split('/')[-3:]) == \"3124/5842/514.dcm\":\n        norm = np.zeros((512,512), dtype=np.uint8)\n    else:\n        dcm = dicomsdl.open(path)\n        pixel_array = dicomsdl_to_numpy_image(dcm)\n        if dcm.PixelRepresentation == 1:\n            bit_shift = dcm.BitsAllocated - dcm.BitsStored\n            dtype = pixel_array.dtype\n            pixel_array = (pixel_array << bit_shift).astype(dtype) >> bit_shift\n\n        #processing\n        pixel_array = pixel_array.astype(np.float32)\n        pixel_array = dcm.RescaleSlope * pixel_array + dcm.RescaleIntercept\n        xmin = dcm.WindowCenter-0.5-(dcm.WindowWidth-1)* 0.5\n        xmax = dcm.WindowCenter-0.5+(dcm.WindowWidth-1)* 0.5\n        norm = np.empty_like(pixel_array, dtype=np.uint8)\n        dicomsdl.util.convert_to_uint8(pixel_array, norm, xmin, xmax)\n\n        if dcm.PhotometricInterpretation == 'MONOCHROME1':\n            norm = 255 - norm\n        # --- (512, 512) np.uint8\n        if img_type == 'seg':\n            norm = cv2.resize(norm, (CFG.image_size_seg[1], CFG.image_size_seg[0]), interpolation=cv2.INTER_LINEAR)\n        elif img_type == 'cls':\n            pass\n        elif img_type == 'extra':\n            norm = (norm - norm.min()) / (norm.max() - norm.min() + 1e-6)\n    return norm \n\n\n\ndef load_dicomsdl_dir(t_paths, img_type):\n    n_scans = len(t_paths)\n    \n    #check inversion\n    min_index, max_index = t_paths[0], t_paths[-1]\n    dcm0 = dicomsdl.open(min_index)\n    dcmN = dicomsdl.open(max_index)\n    sx0, sy0, sz0 = dcm0.ImagePositionPatient\n    sxN, syN, szN = dcmN.ImagePositionPatient\n    \n    inversion = True if szN < sz0 else False\n    if inversion:\n        t_paths = t_paths[::-1]\n        \n    indices = np.quantile(list(range(n_scans)), np.linspace(0., 1., CFG.image_size_seg[2])).round().astype(int)\n    seg_t_paths = [t_paths[i] for i in indices]\n    \n    images = []\n    for filename in seg_t_paths:\n        img = dicomsdl_read_one(filename, img_type)\n        images.append(img)\n\n    images = np.stack(images, -1)\n    \n    images = images - np.min(images)\n    images = images / (np.max(images) + 1e-4)\n    images = (images * 255).astype(np.uint8)\n    return images, t_paths\n\n\nclass SegTestDataset(Dataset):\n\n    def __init__(self, df):\n        self.df = df.reset_index()\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        psid = str(row['psid'])\n        pid, sid = str(row['patient_id']), str(row['series_id'])\n        \n        t_paths = sorted(glob(os.path.join(test_dir, pid, sid, \"*\")), key=lambda x: int(x.split('/')[-1].split(\".\")[0]))\n\n        image, t_paths = load_dicomsdl_dir(t_paths, 'seg') # load : (0,1)\n        \n        if image.ndim < 4:\n            image = np.expand_dims(image, 0)\n        image = image.astype(np.float32).repeat(3, 0)  # to 3ch\n        image = image / 255.\n        \n        return torch.tensor(image).float(), t_paths, psid","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:22.673631Z","iopub.execute_input":"2023-10-21T17:25:22.674492Z","iopub.status.idle":"2023-10-21T17:25:22.694879Z","shell.execute_reply.started":"2023-10-21T17:25:22.674456Z","shell.execute_reply":"2023-10-21T17:25:22.693963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"from timm4smp.models.layers.conv2d_same import Conv2dSame\nfrom conv3d_same import Conv3dSame\n\ndef convert_3d(module):\n\n    module_output = module\n    if isinstance(module, torch.nn.BatchNorm2d):\n        module_output = torch.nn.BatchNorm3d(\n            module.num_features,\n            module.eps,\n            module.momentum,\n            module.affine,\n            module.track_running_stats,\n        )\n        if module.affine:\n            with torch.no_grad():\n                module_output.weight = module.weight\n                module_output.bias = module.bias\n        module_output.running_mean = module.running_mean\n        module_output.running_var = module.running_var\n        module_output.num_batches_tracked = module.num_batches_tracked\n        if hasattr(module, \"qconfig\"):\n            module_output.qconfig = module.qconfig\n            \n    elif isinstance(module, Conv2dSame):\n        module_output = Conv3dSame(\n            in_channels=module.in_channels,\n            out_channels=module.out_channels,\n            kernel_size=module.kernel_size[0],\n            stride=module.stride[0],\n            padding=module.padding[0],\n            dilation=module.dilation[0],\n            groups=module.groups,\n            bias=module.bias is not None,\n        )\n        module_output.weight = torch.nn.Parameter(module.weight.unsqueeze(-1).repeat(1,1,1,1,module.kernel_size[0]))\n\n    elif isinstance(module, torch.nn.Conv2d):\n        module_output = torch.nn.Conv3d(\n            in_channels=module.in_channels,\n            out_channels=module.out_channels,\n            kernel_size=module.kernel_size[0],\n            stride=module.stride[0],\n            padding=module.padding[0],\n            dilation=module.dilation[0],\n            groups=module.groups,\n            bias=module.bias is not None,\n            padding_mode=module.padding_mode\n        )\n        module_output.weight = torch.nn.Parameter(module.weight.unsqueeze(-1).repeat(1,1,1,1,module.kernel_size[0]))\n\n    elif isinstance(module, torch.nn.MaxPool2d):\n        module_output = torch.nn.MaxPool3d(\n            kernel_size=module.kernel_size,\n            stride=module.stride,\n            padding=module.padding,\n            dilation=module.dilation,\n            ceil_mode=module.ceil_mode,\n        )\n    elif isinstance(module, torch.nn.AvgPool2d):\n        module_output = torch.nn.AvgPool3d(\n            kernel_size=module.kernel_size,\n            stride=module.stride,\n            padding=module.padding,\n            ceil_mode=module.ceil_mode,\n        )\n\n    for name, child in module.named_children():\n        module_output.add_module(\n            name, convert_3d(child)\n        )\n    del module\n\n    return module_output","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:22.69591Z","iopub.execute_input":"2023-10-21T17:25:22.696401Z","iopub.status.idle":"2023-10-21T17:25:22.718668Z","shell.execute_reply.started":"2023-10-21T17:25:22.696377Z","shell.execute_reply":"2023-10-21T17:25:22.717795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### segmentation model","metadata":{}},{"cell_type":"code","source":"# segmentation model\n\nclass TimmSegModel(nn.Module):\n    def __init__(self, backbone, segtype='unet', pretrained=False):\n        super(TimmSegModel, self).__init__()\n\n        self.encoder = timm4smp.create_model(\n            backbone,\n            in_chans=3,\n            features_only=True,\n            pretrained=pretrained\n        )\n        g = self.encoder(torch.rand(1, 3, 64, 64))\n        encoder_channels = [1] + [_.shape[1] for _ in g]\n        decoder_channels = [256, 128, 64, 32, 16]\n        if segtype == 'unet':\n            self.decoder = smp.unet.decoder.UnetDecoder(\n                encoder_channels=encoder_channels[:n_blocks+1],\n                decoder_channels=decoder_channels[:n_blocks],\n                n_blocks=n_blocks,\n            )\n        self.segmentation_head = nn.Conv2d(decoder_channels[n_blocks-1], 5, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n\n    def forward(self,x):\n        global_features = [0] + self.encoder(x)[:n_blocks]\n        seg_features = self.decoder(*global_features)\n        seg_features = self.segmentation_head(seg_features)\n        return seg_features\n","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:22.722504Z","iopub.execute_input":"2023-10-21T17:25:22.723348Z","iopub.status.idle":"2023-10-21T17:25:22.731524Z","shell.execute_reply.started":"2023-10-21T17:25:22.723323Z","shell.execute_reply":"2023-10-21T17:25:22.73048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### organ & bowel model","metadata":{}},{"cell_type":"code","source":"# Organ & Bowel Model\n\n    \n# Organ Model\nclass TimmModel(nn.Module):\n    def __init__(self, backbone, pretrained=False):\n        super(TimmModel, self).__init__()\n\n        self.encoder = timm.create_model(\n            backbone,\n            in_chans=6,\n            num_classes=3,\n            features_only=False,\n            drop_rate=0,\n            drop_path_rate=0,\n            pretrained=pretrained\n        )\n\n        if 'efficient' in backbone:\n            hdim = self.encoder.conv_head.out_channels\n            self.encoder.classifier = nn.Identity()\n        elif 'convnext' in backbone:\n            hdim = self.encoder.head.fc.in_features\n            self.encoder.head.fc = nn.Identity()\n        elif 'seresnext' in backbone or 'resnet' in backbone:\n            hdim = self.encoder.fc.in_features\n            self.encoder.fc = nn.Identity()\n\n\n        self.lstm = nn.LSTM(hdim, 256, num_layers=2, dropout=0, bidirectional=True, batch_first=True)\n        self.head = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.Dropout(0),\n            nn.LeakyReLU(0.1),\n            nn.Linear(256, 3),\n        )\n\n    def forward(self, x):  # (bs, nslice, ch, sz, sz)\n        bs, n_slice_per_c, in_chans, image_size, image_size = x.shape\n        \n        x = x.view(bs * n_slice_per_c, in_chans, image_size, image_size)\n        feat = self.encoder(x)\n        feat = feat.view(bs, n_slice_per_c, -1)\n        feat, _ = self.lstm(feat)\n        feat = feat.contiguous().view(bs * n_slice_per_c, -1)\n        feat = self.head(feat) # (2 * n_slice_per_c, 3)\n        feat = feat.view(bs, n_slice_per_c, 3).contiguous()\n        return feat # (bs, 15, 3)\n\n\n# Bowel Model\nclass TimmBowelModel(nn.Module):\n    def __init__(self, backbone, pretrained=False):\n        super(TimmBowelModel, self).__init__()\n\n        self.encoder = timm.create_model(\n            backbone,\n            in_chans=6,\n            num_classes=1, # 1\n            features_only=False,\n            drop_rate=0,\n            drop_path_rate=0,\n            pretrained=pretrained\n        )\n\n        if 'efficient' in backbone:\n            hdim = self.encoder.conv_head.out_channels\n            self.encoder.classifier = nn.Identity()\n        elif 'convnext' in backbone:\n            hdim = self.encoder.head.fc.in_features\n            self.encoder.head.fc = nn.Identity()\n        elif 'seresnext' in backbone or 'resnet' in backbone:\n            hdim = self.encoder.fc.in_features\n            self.encoder.fc = nn.Identity()\n\n\n        self.lstm = nn.LSTM(hdim, 256, num_layers=2, dropout=0, bidirectional=True, batch_first=True)\n        self.image_head = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.Dropout(0),\n            nn.LeakyReLU(0.1),\n            nn.Linear(256, 1)\n        ) # output : (bs * n_slice_per_c, 1)\n\n        self.study_head = nn.Sequential(\n            nn.Linear(1024, 256),\n            nn.BatchNorm1d(256),\n            nn.Dropout(0),\n            nn.LeakyReLU(0.1),\n            nn.Linear(256, 1)\n        )\n\n\n    def forward(self, x):  # (bs, nslice, ch, sz, sz)\n        bs, n_slice_per_c, in_chans, image_size, image_size = x.shape\n        x = x.view(bs * n_slice_per_c, in_chans, image_size, image_size)\n        feat = self.encoder(x)\n        feat = feat.view(bs, n_slice_per_c, -1)\n        feat, _ = self.lstm(feat)\n\n        # image level\n        image_feat = feat.contiguous().view(bs * n_slice_per_c, -1)\n        image_logit = self.image_head(image_feat) # (bs * n_slice_per_c, 1)\n        image_logit = image_logit.view(bs, n_slice_per_c, 1).contiguous() # (bs, n_slice_per_c, 1)\n\n        # study level\n        avg_pool = torch.mean(feat, 1)   # (bs, 512)\n        max_pool = torch.max(feat, 1)[0] # (bs, 512)\n        study_feat = torch.cat((max_pool, avg_pool), 1) # (bs, 1024)\n        study_logit = self.study_head(study_feat)\n        \n        return study_logit, image_logit\n\n","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:22.732684Z","iopub.execute_input":"2023-10-21T17:25:22.732949Z","iopub.status.idle":"2023-10-21T17:25:22.755261Z","shell.execute_reply.started":"2023-10-21T17:25:22.732927Z","shell.execute_reply":"2023-10-21T17:25:22.754428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### extra models (feature extractor + sequence)","metadata":{}},{"cell_type":"code","source":"# Extra Model\nclass Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        # x.shape 1024\n        feature_dim = self.feature_dim # 1024\n        step_dim = self.step_dim # 192 (m_size)\n        \n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)\n        \n        \n# Seghead Model\nclass TimmFeatExtractorSeg(nn.Module):\n    def __init__(self, backbone, decoder_channels, pretrained=False):\n        super(TimmFeatExtractorSeg, self).__init__()\n\n        self.encoder = timm.create_model(backbone, in_chans=5, features_only=True, drop_rate=0, drop_path_rate=0, pretrained=pretrained)\n\n        if 'efficient' in backbone:\n            output_dim = 1280\n        elif 'seresnext50_32x4d' in backbone:\n            output_dim = self.encoder.feature_info.channels()[-1]\n\n        self.decoder = UnetDecoder(encoder_channels=self.encoder.feature_info.channels(), decoder_channels=decoder_channels, n_blocks=5, use_batchnorm=True)\n        self.upsample = nn.UpsamplingBilinear2d(scale_factor=2)\n        self.seg_head = SegmentationHead(in_channels=32, out_channels=1, activation=None, kernel_size=3)\n        \n        self.pool = nn.AdaptiveAvgPool2d((1,1))\n\n        self.study_linear = nn.Sequential(nn.Linear(output_dim, 256), nn.BatchNorm1d(256), nn.Dropout(0), nn.LeakyReLU(0.1), nn.Linear(256, 1))\n        self.image_linear = nn.Sequential(nn.Linear(output_dim, 256), nn.BatchNorm1d(256), nn.Dropout(0), nn.LeakyReLU(0.1), nn.Linear(256, 1))\n        \n    def forward(self, x): \n        features = self.encoder(x) \n        xseg = self.decoder(*features)\n        xseg = self.upsample(xseg)\n        xseg = self.seg_head(xseg) # (bs, 1, 384, 384)\n        xseg = xseg.squeeze(1)\n\n        pool = self.pool(features[-1])\n        pool = pool.squeeze().squeeze()\n\n        study_logit = self.study_linear(pool)\n        image_logit = self.image_linear(pool)\n        return study_logit, image_logit, xseg\n    \n\nclass SeqGRUModel(nn.Module):\n    def __init__(self, config):\n        super(SeqGRUModel, self).__init__()\n        self.lstm1 = nn.GRU(config.emb_dim*3, config.lstm_size, bidirectional=True, batch_first=True)\n        self.lstm2 = nn.GRU(config.lstm_size*2, config.lstm_size, bidirectional=True, batch_first=True)\n        self.image_linear = nn.Linear(config.lstm_size*2, 1)\n\n        self.study_linear = nn.Sequential(nn.Linear(config.lstm_size*4, 256), nn.BatchNorm1d(256), nn.Dropout(0), nn.LeakyReLU(0.1), nn.Linear(256, 1))\n\n        self.attention = Attention(config.lstm_size*2, config.m_size)\n\n    def forward(self, x, mask): \n        # x = SpatialDropout(0.2)(x)\n        feat, _ = self.lstm1(x) # (192, 1024)\n        image_logits = self.image_linear(feat)\n        feat, _ = self.lstm2(feat) # (192, 1024)\n        max_pool, _ = torch.max(feat, 1) # (1024)\n        att_pool = self.attention(feat, mask) # (1024)\n        conc = torch.cat((max_pool, att_pool), 1) # (2048)\n        logits = self.study_linear(conc)\n        return logits, image_logits\n    \n    \nclass TimmEffFeatExtractorSeg(nn.Module):\n    def __init__(self, backbone, decoder_channels, pretrained=False):\n        super(TimmEffFeatExtractorSeg, self).__init__()\n\n        encoder = timm.create_model(\n            backbone,\n            in_chans=5, # 3\n            features_only=False,\n            drop_rate=0,\n            drop_path_rate=0,\n            pretrained=pretrained\n        )\n        self.hidden_layer = nn.Sequential(*list(encoder.children())[-4:-1]) # conv_head, bn, pooling\n        del encoder\n        gc.collect()\n        torch.cuda.empty_cache()\n        \n        self.encoder = timm.create_model(\n            backbone,\n            in_chans=5, # 3\n            features_only=True,\n            drop_rate=0,\n            drop_path_rate=0,\n            pretrained=pretrained\n        )\n\n        if 'efficient' in backbone:\n            output_dim = 1280\n        elif 'seresnext50_32x4d' in backbone:\n            output_dim = self.encoder.feature_info.channels()[-1]\n\n        self.decoder = UnetDecoder(encoder_channels=self.encoder.feature_info.channels(), decoder_channels=decoder_channels, n_blocks=5, use_batchnorm=True)\n        self.upsample = nn.UpsamplingBilinear2d(scale_factor=2)\n        self.seg_head = SegmentationHead(in_channels=48, out_channels=1, activation=None, kernel_size=3)\n        self.pool = nn.AdaptiveAvgPool2d((1,1))\n\n        self.study_linear = nn.Sequential(\n            nn.Linear(output_dim, 256),\n            nn.BatchNorm1d(256),\n            nn.Dropout(0),\n            nn.LeakyReLU(0.1),\n            nn.Linear(256, 1)\n        )\n        self.image_linear = nn.Sequential(\n            nn.Linear(output_dim, 256),\n            nn.BatchNorm1d(256),\n            nn.Dropout(0),\n            nn.LeakyReLU(0.1),\n            nn.Linear(256, 1)\n        )\n        \n        \n    def forward(self, x): \n        features = self.encoder(x) \n        xseg = self.decoder(*features)\n        xseg = self.upsample(xseg)\n        xseg = self.seg_head(xseg) # (bs, 1, 384, 384)\n        xseg = xseg.squeeze(1)\n\n        pool = self.hidden_layer(features[-1])\n\n        study_logit = self.study_linear(pool)\n        image_logit = self.image_linear(pool)\n        return study_logit, image_logit, xseg\n\n","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:22.756494Z","iopub.execute_input":"2023-10-21T17:25:22.756739Z","iopub.status.idle":"2023-10-21T17:25:22.78721Z","shell.execute_reply.started":"2023-10-21T17:25:22.756718Z","shell.execute_reply":"2023-10-21T17:25:22.786424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Models","metadata":{}},{"cell_type":"code","source":"models_seg_all = []\n\nfor seg_config in seg_configs:\n    \n    models_seg = []\n    \n    n_blocks = 4\n    for fold in range(5):\n        model = TimmSegModel(seg_config.backbone, pretrained=False)\n        model = convert_3d(model)\n        model = model.to(device)\n        load_model_file = os.path.join(seg_config.model_dir_seg, f'{seg_config.backbone}/{seg_config.kernel_type}_fold{fold}_best.pth')\n        ###\n        # sd = torch.load(load_model_file, map_location='cpu')\n        sd = torch.load(load_model_file)\n        ###\n        if 'model_state_dict' in sd.keys():\n            sd = sd['model_state_dict']\n        sd = {k[7:] if k.startswith('module.') else k: sd[k] for k in sd.keys()}\n        model.load_state_dict(sd, strict=True)\n        model.eval()\n        models_seg.append(model)\n    \n    models_seg_all.append(models_seg)\n    \ndel models_seg, model, load_model_file, sd\ngc.collect()\n    \nprint('models_seg_all :', len(models_seg_all), '/', len(models_seg_all[0]))","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:22.788139Z","iopub.execute_input":"2023-10-21T17:25:22.788389Z","iopub.status.idle":"2023-10-21T17:25:35.080591Z","shell.execute_reply.started":"2023-10-21T17:25:22.788366Z","shell.execute_reply":"2023-10-21T17:25:35.079621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_organ_1, models_bowel_1 = [], []\nmodels_organ_2, models_bowel_2 = [], []\n\nfor i, ob_config in zip(range(1,3), [OBCFG1, OBCFG2]):\n    \n    models_organ, models_bowel = [], []\n    \n    # organ\n    for fold in range(ob_config.folds):\n        model = TimmModel(ob_config.organ_backbone, pretrained=False)\n        load_model_file = os.path.join(ob_config.organ_model_dir_cls, f'{ob_config.organ_kernel_type}_fold{fold}_best.pth')\n        sd = torch.load(load_model_file)\n        if 'model_state_dict' in sd.keys():\n            sd = sd['model_state_dict']\n        sd = {k[7:] if k.startswith('module.') else k: sd[k] for k in sd.keys()}\n        model.load_state_dict(sd, strict=True)\n        model = model.to(device)\n        model.eval()\n        models_organ.append(model)\n    \n    # bowel\n    for fold in range(ob_config.folds):\n        model = TimmBowelModel(ob_config.bowel_backbone, pretrained=False)\n        load_model_file = os.path.join(ob_config.bowel_model_dir_cls, f'{ob_config.bowel_kernel_type}_fold{fold}_best.pth')\n        sd = torch.load(load_model_file)\n        if 'model_state_dict' in sd.keys():\n            sd = sd['model_state_dict']\n        sd = {k[7:] if k.startswith('module.') else k: sd[k] for k in sd.keys()}\n        model.load_state_dict(sd, strict=True)\n        model = model.to(device)\n        model.eval()\n        models_bowel.append(model)\n    \n    if i == 1:\n        models_organ_1, models_bowel_1 = models_organ, models_bowel\n    if i == 2:\n        models_organ_2, models_bowel_2 = models_organ, models_bowel\n         \n    del models_organ, models_bowel, model, sd\n    gc.collect()\n    torch.cuda.empty_cache()\n\nprint('1:', len(models_organ_1), len(models_bowel_1))\nprint('2:', len(models_organ_2), len(models_bowel_2))\n# print('total:', len(models_organ_1), len(models_bowel_1))\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:25:35.082067Z","iopub.execute_input":"2023-10-21T17:25:35.082447Z","iopub.status.idle":"2023-10-21T17:26:08.46575Z","shell.execute_reply.started":"2023-10-21T17:25:35.082414Z","shell.execute_reply":"2023-10-21T17:26:08.464815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict","metadata":{}},{"cell_type":"code","source":"display(df.head(), df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:26:08.467309Z","iopub.execute_input":"2023-10-21T17:26:08.46759Z","iopub.status.idle":"2023-10-21T17:26:08.480079Z","shell.execute_reply.started":"2023-10-21T17:26:08.467564Z","shell.execute_reply":"2023-10-21T17:26:08.479165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def main():\n    def load_bone(msk, cid, t_paths, config, cropped_images):\n        \"\"\"\n            t_paths : inverse or not\n            x,y,z : segmented mask\n            xx, yy, zz : cropped\n        \"\"\"\n        \n        n_scans = len(t_paths)\n        bone = []\n\n        msk_size = 128\n\n        try:\n            msk_b = msk[cid] > 0.2\n            msk_c = msk[cid] > 0.05\n\n            x = np.where(msk_b.sum(1).sum(1) > 0)[0]\n            y = np.where(msk_b.sum(0).sum(1) > 0)[0]\n            z = np.where(msk_b.sum(0).sum(0) > 0)[0]\n\n            if len(x) == 0 or len(y) == 0 or len(z) == 0:\n                x = np.where(msk_c.sum(1).sum(1) > 0)[0]\n                y = np.where(msk_c.sum(0).sum(1) > 0)[0]\n                z = np.where(msk_c.sum(0).sum(0) > 0)[0]\n\n            x1, x2 = max(0, x[0] - 1), min(msk.shape[1], x[-1] + 1)\n            y1, y2 = max(0, y[0] - 1), min(msk.shape[2], y[-1] + 1)\n            z1, z2 = max(0, z[0] - 1), min(msk.shape[3], z[-1] + 1)\n            zz1, zz2 = int(z1 / 128 * n_scans), int(z2 / 128 * n_scans)\n            # z1 / 128 * 384 = z1 * 3\n\n            if cid != 4:\n                inds = np.linspace(zz1 ,zz2-1, config.n_slice_per_c).astype(int) # 15 slices\n                inds_ = np.linspace(z1 ,z2-1, config.n_slice_per_c).astype(int)\n            else: # bowel : want all slices\n                inds = np.linspace(zz1 ,zz2-1, config.n_slice_per_c * config.mul).astype(int) # 30 slices\n                inds_ = np.linspace(z1 ,z2-1, config.n_slice_per_c * config.mul).astype(int)\n\n\n            if 0:\n                print(f\"{cid} | x {x1} {x2} | y {y1} {y2} | z {z1} {z2} | zz {zz1} {zz2} | n_scans {n_scans}\")\n                print('inds  :', inds)\n                print('inds_ :',inds_)\n\n\n\n            for sid, (ind, ind_) in enumerate(zip(inds, inds_)):\n\n                msk_this = msk[cid, :, :, ind_]\n\n                images = []\n                for i in range(-config.n_ch//2+1, config.n_ch//2+1):\n                    try:\n                        if config.slice_type == 1:\n                            image = dicomsdl_read_one(t_paths[ind+1], 'cls') # uint8 (0,255) (512, 512) (w/o resize) ############################################################## prob\n                        elif config.slice_type == 2:\n                            image = dicomsdl_read_one(t_paths[ind+i], 'cls') \n                        \n                        images.append(image)\n                    except:\n                        # print(f'cid {cid} sid {sid} ch {i} except')\n                        images.append(np.zeros((512, 512)))\n\n                data = np.stack(images, -1)\n                msk_this = msk_this[x1:x2, y1:y2]\n\n                xx1 = int(x1 / msk_size * data.shape[0])\n                xx2 = int(x2 / msk_size * data.shape[0])\n                yy1 = int(y1 / msk_size * data.shape[1])\n                yy2 = int(y2 / msk_size * data.shape[1])\n\n                data = data[xx1:xx2, yy1:yy2]\n\n                data = np.stack(\n                    [cv2.resize(data[:, :, i], (config.image_size_cls, config.image_size_cls), \n                                interpolation = cv2.INTER_LINEAR) \n                        for i in range(5)], -1\n                )\n\n                msk_this = (msk_this * 255).astype(np.uint8)\n                msk_this = cv2.resize(msk_this, (config.image_size_cls, config.image_size_cls), interpolation = cv2.INTER_LINEAR)\n\n                data = np.concatenate([data, msk_this[:, :, np.newaxis]], -1)\n\n                bone.append(torch.tensor(data))\n\n        except:\n            for sid in range(config.n_slice_per_c):\n                bone.append(torch.ones((config.image_size_cls, config.image_size_cls, 5+1)).int())\n\n        cropped_images[cid] = torch.stack(bone, 0)\n\n\n    def load_cropped_images(msk, t_paths, config, n_ch=5):\n        for cid in range(5):\n            threads[cid] = threading.Thread(target=load_bone, args=(msk, cid, t_paths, config, cropped_images))\n            threads[cid].start()\n        for cid in range(5):\n            threads[cid].join()\n\n        return torch.cat(cropped_images, 0)\n\n    \n    ################################# predict organs ##################################################################\n    \n    liver_outputs = []\n    spleen_outputs = []\n    kidney_outputs = []\n    bowel_outputs = []\n    \n    dataset_seg = SegTestDataset(df) # (3, 128, 128, 128)\n    \n    loader_seg = torch.utils.data.DataLoader(\n        dataset_seg, \n        batch_size=1, \n        shuffle=False, \n        num_workers=CFG.num_workers\n    )\n\n    # bs 1 / by psid level\n\n    d = {}\n    \n    with torch.no_grad():\n        for batch_id, (images, t_paths, psid) in tqdm(enumerate(loader_seg), total=len(loader_seg)):\n            images = images.cuda()\n            \n\n            t_paths = np.array(t_paths)\n            if len(t_paths.shape) != 1:\n                t_paths = t_paths.squeeze(-1)\n            \n            d[psid[0]] = t_paths\n\n            ####### 1) Segmentation #######\n            pred_masks = []\n            for models_seg in models_seg_all:\n                pred_mask = []\n                for i, model in enumerate(models_seg):\n\n                    pmask = model(images).sigmoid().float().detach() # (1, 5, 128, 128, 128)\n                    pred_mask.append(pmask)\n\n                pred_mask = torch.stack(pred_mask, 0).mean(0).cpu().numpy()\n                pred_masks.append(pred_mask)\n            pred_masks = np.stack(pred_masks, 0).mean(0) # (1, 5, 128, 128, 128)\n\n            del images\n            gc.collect()\n\n            ####### 2) Build cls input #######\n            \n            _liver_outputs = []\n            _spleen_outputs = []\n            _kidney_outputs = []\n            _bowel_outputs = []\n            \n            \n            for cls_config, cls_organ_models, cls_bowel_models in zip([OBCFG1, OBCFG2], \n                                                                      [models_organ_1, models_organ_2], \n                                                                      [models_bowel_1, models_bowel_2]):\n        \n                cls_inp = []\n                threads = [None] * 5\n                cropped_images = [None] * 5\n\n                for i in range(pred_masks.shape[0]): # 1\n                    row = dataset_seg.df.iloc[batch_id * CFG.batch_size_seg + i]\n                    cropped_images = load_cropped_images(pred_masks[i], t_paths, cls_config) # bigger size\n                    cls_inp.append(cropped_images.permute(0, 3, 1, 2).float() / 255.)\n                cls_inp = torch.stack(cls_inp, 0).to(device)  # e.g. (1, 105, 6, 224, 224)\n\n                del threads, cropped_images\n                gc.collect()\n\n\n                ####### 2-1) organ models #######\n                liver_preds, spleen_preds, kidney_preds = [], [], []\n\n                liver = cls_inp[0,0:15,:,:,:]\n                spleen = cls_inp[0,15:30,:,:,:]\n                l_kidney = cls_inp[0,30:45,:,:,:]\n                r_kidney = cls_inp[0,45:60,:,:,:]\n\n                kidney = []\n                for ind in range(15):\n                    _kidney = torch.cat(\n                        [\n                            F.interpolate(l_kidney[ind].unsqueeze(0), (l_kidney.shape[-2], l_kidney.shape[-1]//2), mode='bilinear'),\n                            F.interpolate(r_kidney[ind].unsqueeze(0), (r_kidney.shape[-2], r_kidney.shape[-1]//2), mode='bilinear')\n                        ], dim=-1)\n                    kidney.append(_kidney)\n                kidney = torch.cat(kidney, dim=0)\n\n                for i, model in enumerate(cls_organ_models):\n\n                    liver_logits = model(liver.unsqueeze(0))\n                    spleen_logits = model(spleen.unsqueeze(0))\n                    kidney_logits = model(kidney.unsqueeze(0))\n\n                    liver_preds.append(F.softmax(liver_logits, dim=-1).detach().cpu().numpy())\n                    spleen_preds.append(F.softmax(spleen_logits, dim=-1).detach().cpu().numpy())\n                    kidney_preds.append(F.softmax(kidney_logits, dim=-1).detach().cpu().numpy())\n                \n                del liver, spleen, l_kidney, r_kidney, kidney\n                gc.collect()\n                \n                ####### 2-2) bowel models #######\n                bowel_preds = []\n\n                bowel = cls_inp[0,60:,:,:,:]\n                \n                for i, model in enumerate(cls_bowel_models):\n                    bowel_logits, _ = model(bowel.unsqueeze(0))\n                    bowel_preds.append(bowel_logits.sigmoid().detach().cpu().numpy())\n\n                del bowel\n                gc.collect()\n                \n                _liver_outputs.append(np.mean(liver_preds, axis=0)) # append (1,15,3)\n                _spleen_outputs.append(np.mean(spleen_preds, axis=0))\n                _kidney_outputs.append(np.mean(kidney_preds, axis=0))\n                _bowel_outputs.append(np.mean(bowel_preds))\n\n            liver_outputs.append(_liver_outputs[0] * 0.8 + _liver_outputs[1] * 0.2)\n            spleen_outputs.append(_spleen_outputs[0] * 0.8 + _spleen_outputs[1] * 0.2)\n            kidney_outputs.append(_kidney_outputs[0] * 0.8 + _kidney_outputs[1] * 0.2)\n            bowel_outputs.append(_bowel_outputs[0] * 0.8 + _bowel_outputs[1] * 0.2)        \n    # output : (n, 15, 3)\n    return liver_outputs, spleen_outputs, kidney_outputs, bowel_outputs, d\n            \nif __name__ == '__main__':\n    if not DEBUG and not fast_sub:\n    # if DEBUG:\n        liver_outputs, spleen_outputs, kidney_outputs, bowel_outputs, d = main()\n\n        del models_organ_1, models_bowel_1, models_organ_2, models_bowel_2, models_seg_all\n        gc.collect()\n        torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:26:08.481606Z","iopub.execute_input":"2023-10-21T17:26:08.482236Z","iopub.status.idle":"2023-10-21T17:26:08.527424Z","shell.execute_reply.started":"2023-10-21T17:26:08.482202Z","shell.execute_reply":"2023-10-21T17:26:08.526526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extra","metadata":{}},{"cell_type":"code","source":"# seghead extra model\nif 1:\n    models_feat_1, models_seq_1 = [], []\n\n    for i, extra_config in zip(range(1,2), [ExtraCFG1]):\n\n        models_feat, models_seq = [], []\n\n        # feature\n        for fold in range(5):\n            model = TimmFeatExtractorSeg(extra_config.feature_backbone, (256, 128, 64, 32, 16), pretrained=False)\n            model = model.to(device)\n            load_model_file = os.path.join(extra_config.feature_model_dir, f'{extra_config.feature_kernel_type}', f'extra-feat-sliding-seg_fold{fold}_best.pth')\n            sd = torch.load(load_model_file)\n            # sd = torch.load(load_model_file, map_location='cpu')\n            if 'model_state_dict' in sd.keys():\n                sd = sd['model_state_dict']\n            sd = {k[7:] if k.startswith('module.') else k: sd[k] for k in sd.keys()}\n            model.load_state_dict(sd, strict=True)\n            model.eval()\n            models_feat.append(model)\n\n        # sequence\n        for fold in range(5):\n            model = SeqGRUModel(extra_config)\n            model = model.to(device)\n            load_model_file = os.path.join(extra_config.sequence_model_dir, f'{extra_config.sequence_kernel_type}',f'extra-feat-sliding-seg-seq_fold{fold}_best.pth')\n            sd = torch.load(load_model_file)\n            if 'model_state_dict' in sd.keys():\n                sd = sd['model_state_dict']\n            sd = {k[7:] if k.startswith('module.') else k: sd[k] for k in sd.keys()}\n            model.load_state_dict(sd, strict=True)\n            model.eval()\n            models_seq.append(model)\n\n        if i == 1:\n            models_feat_1, models_seq_1 = models_feat, models_seq\n\n        del models_feat, models_seq, model, sd\n        gc.collect()\n        torch.cuda.empty_cache()\n    \n    print(1, len(models_feat_1), len(models_seq_1))","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:26:08.528684Z","iopub.execute_input":"2023-10-21T17:26:08.529039Z","iopub.status.idle":"2023-10-21T17:26:23.032422Z","shell.execute_reply.started":"2023-10-21T17:26:08.529007Z","shell.execute_reply":"2023-10-21T17:26:23.031397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# seghead extra model EfficientNet\nif 0:\n    models_feat_2, models_seq_2 = [], []\n\n    for i, extra_config in zip(range(1,2), [ExtraCFG2]):\n\n        models_feat, models_seq = [], []\n\n        # feature\n        for fold in range(5):\n            model = TimmEffFeatExtractorSeg(extra_config.feature_backbone, (256, 160, 64, 48, 24), pretrained=False)\n            model = model.to(device)\n            load_model_file = os.path.join(extra_config.feature_model_dir, f'{extra_config.feature_kernel_type}', f'extra-feat-sliding-seg_fold{fold}_best.pth')\n            sd = torch.load(load_model_file)\n            # sd = torch.load(load_model_file, map_location='cpu')\n            if 'model_state_dict' in sd.keys():\n                sd = sd['model_state_dict']\n            sd = {k[7:] if k.startswith('module.') else k: sd[k] for k in sd.keys()}\n            model.load_state_dict(sd, strict=True)\n            model.eval()\n            models_feat.append(model)\n\n        # sequence\n        for fold in range(5):\n            model = SeqGRUModel(extra_config)\n            model = model.to(device)\n            load_model_file = os.path.join(extra_config.sequence_model_dir, f'{extra_config.sequence_kernel_type}',f'extra-feat-sliding-seg-seq_fold{fold}_best.pth')\n            sd = torch.load(load_model_file)\n            if 'model_state_dict' in sd.keys():\n                sd = sd['model_state_dict']\n            sd = {k[7:] if k.startswith('module.') else k: sd[k] for k in sd.keys()}\n            model.load_state_dict(sd, strict=True)\n            model.eval()\n            models_seq.append(model)\n\n        if i == 1:\n            models_feat_2, models_seq_2 = models_feat, models_seq\n\n        del models_feat, models_seq, model, sd\n        gc.collect()\n        torch.cuda.empty_cache()\n    \n    print(2, len(models_feat_2), len(models_seq_2))","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:26:23.033526Z","iopub.execute_input":"2023-10-21T17:26:23.033788Z","iopub.status.idle":"2023-10-21T17:26:23.046512Z","shell.execute_reply.started":"2023-10-21T17:26:23.033765Z","shell.execute_reply":"2023-10-21T17:26:23.045621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_extra(t_paths, config, slide=5):\n    indices = [i for i in range(2,len(t_paths)-2) if i%slide == 2]\n    imgs = []\n    for i in indices:\n        x0 = dicomsdl_read_one(t_paths[i-2], 'extra')\n        x1 = dicomsdl_read_one(t_paths[i-1], 'extra')\n        x2 = dicomsdl_read_one(t_paths[i],   'extra') # (0,1) (512, 512)\n        x3 = dicomsdl_read_one(t_paths[i+1], 'extra')\n        x4 = dicomsdl_read_one(t_paths[i+2], 'extra')\n\n        x0 = np.expand_dims(x0, axis=2)\n        x1 = np.expand_dims(x1, axis=2)\n        x2 = np.expand_dims(x2, axis=2)\n        x3 = np.expand_dims(x3, axis=2)\n        x4 = np.expand_dims(x4, axis=2)\n        img = np.concatenate([x0, x1, x2, x3, x4], axis=-1) # (512, 512, 5)\n                \n        img = cv2.resize(\n            img, \n            (config.image_size_extra, config.image_size_extra), interpolation=cv2.INTER_LINEAR\n        ) # (384, 384, 5)\n        \n        imgs.append(torch.tensor(img.transpose(2,0,1)).float())\n        \n    imgs = torch.stack(imgs, 0) # (len, 5, 384, 384)\n    \n    return imgs    \n\nclass ExtraTestDataset(Dataset):\n    def __init__(self, df, d, config):\n        self.df = df.reset_index()\n        self.d = d\n        self.config = config\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        psid = str(row['psid'])\n        t_paths = self.d[psid]\n        images = load_extra(t_paths, self.config, slide=5)\n        return torch.tensor(images).float(), list(t_paths) # (len, 5, 224, 224)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:26:23.047874Z","iopub.execute_input":"2023-10-21T17:26:23.048219Z","iopub.status.idle":"2023-10-21T17:26:23.06656Z","shell.execute_reply.started":"2023-10-21T17:26:23.048185Z","shell.execute_reply":"2023-10-21T17:26:23.065765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extra_main_seghead(df, d, config, models_feat, models_seq):\n    extra_bs = config.extra_bs\n    m_size = config.m_size\n    \n    extra_outputs = []\n\n    extra_ds = ExtraTestDataset(df, d, config)\n    loader_extra = torch.utils.data.DataLoader(extra_ds, batch_size=1, shuffle=False, num_workers=CFG.num_workers)\n\n    with torch.no_grad():\n        for batch_id, (imgs, t_paths) in tqdm(enumerate(loader_extra), total=len(loader_extra)):\n            imgs = imgs[0]\n            \n            # make group for batch input\n            group, remain = imgs.shape[0]//extra_bs, imgs.shape[0]%extra_bs\n\n            embs = []\n            for g in range(group):\n                img_batch = imgs[g * extra_bs : g * extra_bs + extra_bs]\n                if len(img_batch.shape) < 4:\n                    img_batch = torch.unsqueeze(img_batch, dim=0)\n                    \n                _embs = []\n                for i, model in enumerate(models_feat):\n\n#                     if i == fold:\n                    if config.forward_type == 1:\n                        emb = model.encoder(img_batch.cuda())\n                        emb = model.pool(emb[-1]).squeeze().squeeze() # (bs, emb_size)\n                    elif config.forward_type == 2:\n                        emb = model.encoder(img_batch.cuda())\n                        emb = model.hidden_layer(emb[-1])\n                    _embs.append(emb)\n\n                _embs = torch.stack(_embs, 0) # (n_models, bs, emb_size)\n                _embs = torch.mean(_embs, 0) # (bs, emb_size)\n                embs.append(_embs)\n\n            if remain != 0:\n                img_batch = imgs[group * extra_bs:]\n                if len(img_batch.shape) < 4:\n                    img_batch = torch.unsqueeze(img_batch, dim=0)\n                _embs = []\n                for i, model in enumerate(models_feat):\n\n#                     if i == fold:\n                    if config.forward_type == 1:\n                        emb = model.encoder(img_batch.cuda())\n                        emb = model.pool(emb[-1]).squeeze().squeeze() # (bs, emb_size)\n                    elif config.forward_type == 2:\n                        emb = model.encoder(img_batch.cuda())\n                        emb = model.hidden_layer(emb[-1])\n                    _embs.append(emb)\n\n                _embs = torch.stack(_embs, 0) # (n_models, bs, emb_size)\n                if len(_embs.shape) == 2:\n                    _embs = _embs.unsqueeze(1)\n                _embs = torch.mean(_embs, 0) # (bs, emb_size)\n                embs.append(_embs)\n\n            embs = torch.cat(embs).detach().cpu().numpy() # (seq_len, emb_size)\n            \n            del _embs, img_batch, imgs\n            gc.collect()\n\n            # make data for seq model\n            if len(embs) < m_size:\n                pad_sz = (m_size - len(embs))\n                pad = np.zeros((pad_sz, embs.shape[-1]))\n                mask = np.concatenate([np.zeros(pad_sz,), np.ones(len(embs),)])\n                embs = np.concatenate([pad, embs])\n            else: # resize\n                embs = cv2.resize(embs, (embs.shape[-1], m_size), interpolation=cv2.INTER_LINEAR)\n                mask = np.ones(len(embs),)\n\n            diff1 = np.zeros_like(embs)\n            diff1[1:] = embs[1:] - embs[:-1]\n            diff2 = np.zeros_like(embs)\n            diff2[:-1] = embs[:-1] - embs[1:]\n\n            embs = np.concatenate([embs, diff1, diff2], axis=-1)\n\n            embs = torch.tensor(embs).float().unsqueeze(0) # (m_size, emb_dim * 3) (1, 192, 6144)\n            mask = torch.tensor(mask).float().unsqueeze(0) # (m_size)\n\n            ####### 3-2) sequence model #######\n            extra_preds = []\n            for i, model in enumerate(models_seq):\n\n#                 if i == fold:\n                logit, _ = model(embs.to(device), mask.to(device))\n                extra_preds.append(logit.sigmoid().detach().cpu().numpy())\n            \n            # print('extra seq models done')\n\n            extra_outputs.append(np.mean(extra_preds))\n            del embs, mask, diff1, diff2, extra_preds\n            gc.collect()\n            \n    return extra_outputs\n            \nif __name__ == \"__main__\":\n    if not DEBUG and not fast_sub:\n    # if DEBUG:\n        extra_outputs_1 = extra_main_seghead(df, d, ExtraCFG1, models_feat_1, models_seq_1)\n        # extra_outputs_2 = extra_main_seghead(df, d, ExtraCFG2, models_feat_2, models_seq_2)\n        # del ExtraCFG1, models_feat_1, models_seq_1, ExtraCFG2, models_feat_2, models_seq_2\n        del ExtraCFG1, models_feat_1, models_seq_1\n        # extra_outputs = np.array(extra_outputs_1) * 0.5 + np.array(extra_outputs_2) * 0.5\n        extra_outputs = extra_outputs_1\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:26:23.067644Z","iopub.execute_input":"2023-10-21T17:26:23.06794Z","iopub.status.idle":"2023-10-21T17:26:23.090765Z","shell.execute_reply.started":"2023-10-21T17:26:23.067915Z","shell.execute_reply":"2023-10-21T17:26:23.090043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not DEBUG and not fast_sub:\n# if DEBUG:\n    liver_outputs = np.mean(np.stack(liver_outputs), axis=-2).squeeze(1)\n    spleen_outputs = np.mean(np.stack(spleen_outputs), axis=-2).squeeze(1)\n    kidney_outputs = np.mean(np.stack(kidney_outputs), axis=-2).squeeze(1)\n    bowel_outputs = np.stack(bowel_outputs)\n    extra_outputs = np.stack(extra_outputs)\n\n    tmp = pd.DataFrame({'psid':df['psid'].values})\n    tmp['patient_id'] = tmp['psid'].apply(lambda x: int(x.split('_')[0]))\n    tmp['series_id'] = tmp['psid'].apply(lambda x: int(x.split('_')[1]))\n    tmp['bowel_injury'] = bowel_outputs\n    tmp['bowel_healthy'] = 1-tmp['bowel_injury']\n    tmp['extravasation_injury'] = extra_outputs * 3\n    tmp['extravasation_healthy'] = 1-extra_outputs\n    tmp[['kidney_healthy','kidney_low','kidney_high']] = kidney_outputs\n    tmp[['spleen_healthy','spleen_low','spleen_high']] = spleen_outputs\n    tmp[['liver_healthy','liver_low','liver_high']] = liver_outputs\n    tmp = tmp.drop(columns=['psid','series_id'])\n    \n    sub = tmp.groupby('patient_id').mean().reset_index()\n\n    sub.to_csv(\"submission.csv\", index=False)\n    print('sub')\n    \nelif not DEBUG and fast_sub:\n    print('fast_sub')\n    sub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:26:23.091761Z","iopub.execute_input":"2023-10-21T17:26:23.092048Z","iopub.status.idle":"2023-10-21T17:26:23.113716Z","shell.execute_reply.started":"2023-10-21T17:26:23.092025Z","shell.execute_reply":"2023-10-21T17:26:23.112781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:26:23.114927Z","iopub.execute_input":"2023-10-21T17:26:23.115208Z","iopub.status.idle":"2023-10-21T17:26:23.133024Z","shell.execute_reply.started":"2023-10-21T17:26:23.115185Z","shell.execute_reply":"2023-10-21T17:26:23.132172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}