{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os, glob, pickle, time, gc, copy, sys, multiprocessing\nfrom joblib import Parallel, delayed\n\nimport warnings\nimport cv2, pydicom\nwarnings.filterwarnings('ignore')\npd.set_option('display.max_columns', 100) # 表示できる列数","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.parallel\nimport torch.backends.cudnn as cudnn\nimport torch.optim as optim\nimport torch.utils.data\nimport torchvision.transforms as transforms\nimport torchvision.datasets as datasets\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import _LRScheduler\nfrom sklearn import metrics\n\n\nsys.path.append('../input/timm-efficientnet/pytorch-image-models-master/') # Done path OK\nsys.path.append('../input/pretrainedmodels/pretrained-models.pytorch-master/') # Done path OK\nimport timm\nimport pretrainedmodels\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ntorch.backends.cudnn.benchmark = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# test data loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"col_index = 'SOPInstanceUID'\ncol_groupby = 'StudyInstanceUID'\ndf_test_path = \"../input/rsna-str-pulmonary-embolism-detection/test.csv\"\n# df_test_path = \"../input/rsna-str-pulmonary-embolism-detection/train.csv\"\ndf_train_path = \"../input/rsna-str-pulmonary-embolism-detection/train.csv\"\ndf_sub_path = \"../input/rsna-str-pulmonary-embolism-detection/sample_submission.csv\"\ntest_image_path = \"../input/rsna-str-pulmonary-embolism-detection/test/\"\n# test_image_path = \"../input/rsna-str-pulmonary-embolism-detection/train/\"\ntrain_image_path = \"../input/rsna-str-pulmonary-embolism-detection/train/\"\nLEN_PUBLIC = 146853\nDEBUG = False\nnum_cpu = multiprocessing.cpu_count()\nnum_features1_1 = 1408\nweight_dir1_1 = \"../input/rsna2020-1/201024_5_b2_1loss_512_fp16_bs80_lr1e3/201024_5_b2_1loss_512_fp16_bs80_lr1e3\"\nweight_dir2_1 = \"../input/rsna2020-1/201026_3_2ndNNs_features_pool_flip_b0b2/201026_3_2ndNNs_features_pool_flip_b0b2\"\ndo_full = False\nBATCH_SIZE = 64\nNUM_FOLD = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col_targets = [\n    'negative_exam_for_pe',\n    'indeterminate',\n    'chronic_pe',\n    'acute_and_chronic_pe',\n    'central_pe',\n    'leftsided_pe',\n    'rightsided_pe',\n    'rv_lv_ratio_gte_1',\n    'rv_lv_ratio_lt_1',\n    'pe_present_on_image',\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.exists('../input/rsna-str-pulmonary-embolism-detection/train') and not do_full:\n    df_test_full=pd.read_csv(df_test_path)\n    df_test_full_study = df_test_full[df_test_full[col_groupby].duplicated()==False]\n    df_test=pd.read_csv(df_test_path).head(2000)\nelse:\n    df_test_full=pd.read_csv(df_test_path)\n    df_test_full_study = df_test_full[df_test_full[col_groupby].duplicated()==False]\n    df_test=pd.read_csv(df_test_path)\n# df_test=pd.read_csv(df_test_path)\n# df_test=pd.read_csv(df_test_path).head(2000)\nprint(df_test.shape)\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = df_test.sort_values([col_groupby, col_index]).reset_index(drop=True)\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['dicom_path'] = test_image_path + \"/\" +\\\n    df_test[col_groupby].values + \"/\" + \\\n    df_test['SeriesInstanceUID'].values + \"/\" + \\\n    df_test[col_index].values + \".dcm\"\nprint(df_test['dicom_path'][0])\n# df_test['dicom_path'][0] = \"asdfja:sdfk\"\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_test_study = df_test[df_test[col_groupby].duplicated()==False]\ndf_test_study['start_index'] = df_test_study.index.values\ndf_tmp = df_test.groupby(col_groupby)[col_index].agg(len).reset_index()\ndf_tmp.columns = [col_groupby, 'num_images']\ndf_test_study = pd.merge(df_test_study, df_tmp, on=col_groupby)\n# df_test_study = df_test_study.reset_index(drop=True)\n# df_test_study = pd.concat([df_test_study, df_test_study, df_test_study, df_test_study]).reset_index()\nprint(df_test_study.shape)\ndf_test_study.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DicomDataset(Dataset):\n    def __init__(self, X_study, X_image, transform=None, meta=False, verbose=False):\n        self.X_study = X_study\n        self.X_image = X_image\n        self.transform = transform\n        self.verbose = verbose\n\n    def __getitem__(self, index):\n        # get df_study\n        study = self.X_study[col_groupby][index]\n        start_index = self.X_study['start_index'][index]\n        end_index = self.X_study['start_index'][index] + self.X_study['num_images'][index]\n        df_study = self.X_image.iloc[start_index:end_index].reset_index(drop=True)\n\n        # load dicoms\n        images_study = []\n        z_pos = []\n        for i in range(len(df_study)):\n            tmp_path = df_study['dicom_path'][i]\n            try:\n                tmp_dcm = pydicom.dcmread(tmp_path)\n                tmp_npy = np.asarray(tmp_dcm.pixel_array)\n                images_study.append(tmp_npy)\n                if i==0:\n                    RescaleSlope = tmp_dcm['RescaleSlope'].value\n                    RescaleIntercept = tmp_dcm['RescaleIntercept'].value\n                    PatientPosition = tmp_dcm['PatientPosition'].value\n                z_pos.append(tmp_dcm['ImagePositionPatient'].value[-1])\n            except:\n                print(\"loading error!!!, study: {}, index: {}\".format(study, i))\n                tmp_npy = np.zeros([512, 512], np.int16)\n                images_study.append(tmp_npy)\n                if i==0:\n                    RescaleSlope = 1\n                    RescaleIntercept = -1024\n                    PatientPosition = 'HFS'\n                z_pos.append(-10000-i)\n                \n        images_study = np.array(images_study)\n        z_pos = np.array(z_pos)\n        images_study = images_study[np.argsort(z_pos)]\n        df_study['z_pos'] = z_pos\n        df_study = df_study.sort_values('z_pos').reset_index(drop=True)\n        df_study['series_index'] = np.arange(len(df_study))\n        if self.verbose: print(images_study.shape)\n        if self.verbose: print(z_pos)\n        if self.verbose: print(RescaleIntercept, RescaleSlope, PatientPosition)\n            \n        # process images\n        images_study_processed = (images_study.astype(np.float32) * RescaleSlope + RescaleIntercept)/1000\n        if PatientPosition=='FFP':\n            images_study_processed = images_study_processed[:, ::-1, ::-1]\n        images_study_processed = images_study_processed.reshape([-1, 1, 512, 512]).astype(np.float16)\n        \n        return images_study_processed, df_study\n    \n    def __len__(self):\n        return len(self.X_study)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def my_collate(batch):\n    return torch.Tensor(batch[0][0]), batch[0][1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class nnWindow(nn.Module):\n    def __init__(self):\n        super(nnWindow, self).__init__()\n        wso = np.array(((40,80),(80,200),(40,400)))/1000\n        conv_ = nn.Conv2d(1,3, kernel_size=(1, 1))\n        conv_.weight.data.copy_(torch.tensor([[[[1./wso[0][1]]]],[[[1./wso[1][1]]]],[[[1./wso[2][1]]]]]))\n        conv_.bias.data.copy_(torch.tensor([0.5 - wso[0][0]/wso[0][1],\n                                            0.5 - wso[1][0]/wso[1][1],\n                                            0.5 -wso[2][0]/wso[2][1]]))\n        self.window = nn.Sequential(\n            conv_,\n            nn.Sigmoid(),\n            nn.InstanceNorm2d(3)\n        )\n    def forward(self, input1):\n        return self.window(input1)\n        \n        \nclass MyEffNet_b0(nn.Module):\n    def __init__(self, num_classes=10, base_model='tf_efficientnet_b0_ns'):\n        super(MyEffNet_b0, self).__init__()\n\n        self.num_classes = num_classes\n        self.mode = 'train'\n        self.window = nnWindow()\n#         self.base_model = pretrainedmodels.__dict__['resnet18'](num_classes=1000, pretrained='imagenet')\n        self.base_model = timm.create_model(base_model, pretrained=False, num_classes=10).to(device, non_blocking=True)\n        self.avgpool = nn.AdaptiveAvgPool2d(1)\n#         self.last_linear = nn.Linear(512, num_classes+1)\n        self.last_linear = nn.Linear(self.base_model.num_features, num_classes)\n\n    def forward(self, input1):\n        bs, ch, h, w = input1.size()\n        x = self.window(input1)\n        x = self.base_model.forward_features(x) #; print('layer conv1 ',x.size()) # [8, 64, 112, 112]\n        feature = self.avgpool(x).view(bs, -1)\n        y = self.last_linear(feature)\n\n        return y\n\n    def feature(self, input1):\n        bs, ch, h, w = input1.size()\n        x = self.window(input1)\n        x = self.base_model.forward_features(x) #; print('layer conv1 ',x.size()) # [8, 64, 112, 112]\n        feature = self.avgpool(x).view(bs, -1)\n        y = self.last_linear(feature)\n\n        return y, feature","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ngdcm\ndcm読むときにtry except\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SEModule(nn.Module):\n\n    def __init__(self, channels, reduction):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n        self.fc1 = nn.Conv1d(channels, channels // reduction, kernel_size=1,\n                             padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv1d(channels // reduction, channels, kernel_size=1,\n                             padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x\n    \nclass CNN1D(nn.Module):\n\n    def __init__(self, num_classes=400, input_ch=1, verbose=False):\n\n        super(CNN1D, self).__init__()\n        pool = 4\n        drop = 0.1\n        self.verbose = verbose\n        self.layer1 = nn.Sequential(\n                nn.Conv1d(input_ch//pool, 64, kernel_size=7, stride=1, padding=3, bias=False),\n                nn.BatchNorm1d(64),\n                nn.ReLU(inplace=True),\n                SEModule(64, 16),\n#                 nn.Dropout(drop),\n        )\n        self.fpool = nn.MaxPool1d(kernel_size=pool, stride=pool, padding=0)\n        self.maxpool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1)\n#         self.upsample = nn.Upsample(scale_factor=2, mode='bilinear')\n        self.layer2 = nn.Sequential(\n                nn.Conv1d(64, 128, kernel_size=3, stride=1, padding=1, bias=False),\n                nn.BatchNorm1d(128),\n                nn.ReLU(inplace=True),\n                SEModule(128, 16),\n#                 nn.Dropout(drop),\n        )\n        self.layer3 = nn.Sequential(\n                nn.Conv1d(128, 256, kernel_size=3, stride=1, padding=1, bias=False),\n                nn.BatchNorm1d(256),\n                nn.ReLU(inplace=True),\n                SEModule(256, 16),\n#                 nn.Dropout(drop),\n        )\n        self.layer4 = nn.Sequential(\n                nn.Conv1d(256, 512, kernel_size=3, stride=1, padding=1, bias=False),\n                nn.BatchNorm1d(512),\n                nn.ReLU(inplace=True),\n                SEModule(512, 16),\n#                 nn.Dropout(drop),\n        )\n        self.avgpool = nn.AdaptiveAvgPool1d(1)\n        self.fc2 = nn.Conv1d(\n            input_ch//pool+64+128+256+512, \n            2, kernel_size=1)\n#         self.fc = nn.Linear(512, 9)\n        self.fc = nn.Sequential(\n                nn.Linear(512, 512),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n                nn.Linear(512, 512),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n                nn.Linear(512, 9),\n        )\n\n    def forward(self, x_input):\n        bs, ch, d = x_input.size()\n        x0 = torch.transpose(x_input, 1, 2)\n        x0 = self.fpool(x0)\n        x0 = torch.transpose(x0, 1, 2)\n        x1 = self.layer1(x0)\n        x1 = self.maxpool(x1)\n\n        x2 = self.layer2(x1)\n        x2 = self.maxpool(x2)\n        x3 = self.layer3(x2)\n        x3 = self.maxpool(x3)\n        x4 = self.layer4(x3)\n        \n#         tmp = F.adaptive_avg_pool1d(x1, d)\n#         print(tmp.shape)\n#         tmp = F.adaptive_avg_pool1d(x2, d)\n#         print(tmp.shape)\n        x5 = torch.cat([\n            x0,\n            F.adaptive_avg_pool1d(x1, d), \n            F.adaptive_avg_pool1d(x2, d), \n            F.adaptive_avg_pool1d(x3, d), \n            F.adaptive_avg_pool1d(x4, d), \n        ], axis=1)\n        y2 = self.fc2(x5)\n        \n        b, ch, d = x_input.size()\n#         x1 = self.fc(x)\n#         x1 = x1.view(b, -1, 1)\n            \n        y = self.avgpool(x4)\n        y = y.view(b, -1)\n        y = self.fc(y)\n        return y, y2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_test = DicomDataset(df_test_study, df_test)\ntest_loader = DataLoader(\n    dataset_test,\n    batch_size=1,\n    shuffle=False,\n    num_workers=num_cpu,\n    pin_memory=True,\n    collate_fn=my_collate\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_b0 = MyEffNet_b0(base_model='tf_efficientnet_b2_ns').to(device, non_blocking=True)\nmodel_1dcnn = CNN1D(input_ch=num_features1_1).to(device, non_blocking=True)\nmodel_b0.eval()\nmodel_1dcnn.eval()\nlastfunc = nn.Sigmoid().to(device, non_blocking=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_padding(batch):\n    bs, ch, d = batch.shape\n    d_new = int(np.ceil(d/64)*64)\n#     d_new = int(np.ceil(1083/64)*64)\n    batch_new = torch.from_numpy(np.zeros([bs, ch, d_new], np.float32)).to(device, non_blocking=True)\n    batch_new[:, :, :d] = batch\n    return batch_new","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pred_image = []\ndf_pred_study = []\nstarttime = time.time()\nverbose = False\nfor study_index, (images, df_study) in enumerate(test_loader):\n#     if study_index>=100: break\n    if verbose: print(\"load index {} done\".format(study_index), time.time()-starttime)\n    if (study_index+1)%10==0:\n        print(\"{}/{}, sec: {:.1f}\".format(study_index+1, len(df_test_study), time.time()-starttime))\n#     if study_index>10: break\n    num_batches = int(np.ceil(images.shape[0]/BATCH_SIZE))\n    num_images = len(df_study)\n    df_study_image = df_study[[col_groupby, col_index, 'SeriesInstanceUID']]\n    df_study_study = df_study[[col_groupby]].iloc[:1]\n    for fold in range(NUM_FOLD):\n#         if verbose: print(\"load weight start\", time.time()-starttime)\n        model_b0.load_state_dict(torch.load(\"{}/weight_epoch_16_fold{}.pth\".format(weight_dir1_1, fold+1)))\n        model_1dcnn.load_state_dict(torch.load(\"{}/cnn_weight_best_fold{}.pth\".format(weight_dir2_1, fold+1)))\n#         if verbose: print(\"load weight done\", time.time()-starttime)\n        features = []\n        for batch_index in range(num_batches):\n            with torch.no_grad():\n                with torch.cuda.amp.autocast():\n                    batch = images[batch_index*BATCH_SIZE:(batch_index+1)*BATCH_SIZE].to(device, non_blocking=True)\n                    _, feature = model_b0.feature(batch)\n#             print(feature.dtype)\n            features.append(feature)\n        features = torch.cat(features, axis=0) # bs=d, ch\n        features = torch.transpose(features, 0,1).reshape([1, num_features1_1, -1])\n        features = batch_padding(features)\n        with torch.no_grad():\n#             with torch.cuda.amp.autocast():\n            output1, output2 = model_1dcnn(features)\n            output2 = output2[:,-1:]\n            output1 = lastfunc(output1)\n            output2 = lastfunc(output2)[:,:,:num_images]\n        for i, col in enumerate(col_targets[:-1]):\n            df_study_study[\"{}_pred_fold{}\".format(col, fold+1)] = output1[0, i].data.cpu().numpy()\n        df_study_image[\"{}_pred_fold{}\".format(col_targets[-1], fold+1)] = output2[0, 0].data.cpu().numpy()\n    df_pred_study.append(df_study_study)\n    df_pred_image.append(df_study_image)\n# 10/650, sec: 45.6 目安","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Postprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pred_image = pd.concat(df_pred_image).reset_index(drop=True)\ndf_pred_study = pd.concat(df_pred_study).reset_index(drop=True)\nprint(df_pred_image.shape, df_pred_study.shape)\ndf_pred_image.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pred_study.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in col_targets[:-1]:\n    cols_tmp = []\n    for fold in range(NUM_FOLD):\n        cols_tmp.append(\"{}_pred_fold{}\".format(col, fold+1))\n    df_pred_study[col] = df_pred_study[cols_tmp].values.mean(axis=1)\ndf_pred_study.head(30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols_tmp = []\nfor fold in range(NUM_FOLD):\n    cols_tmp.append(\"{}_pred_fold{}\".format(col_targets[-1], fold+1))\ndf_pred_image[col_targets[-1]] = df_pred_image[cols_tmp].values.mean(axis=1)\ndf_pred_image.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# process conflict\n\ndef solve_conflict(df_pred_s, df_pred, TH_NEGATIVE=0.5, TH_INDETERMINATE = 0.5, verbose=True):\n    index_indeterminate = df_pred_s['indeterminate']>TH_INDETERMINATE\n    index_negative = (index_indeterminate==False) & (df_pred_s['negative_exam_for_pe']>TH_NEGATIVE)\n    index_positive = (index_indeterminate==False) & (index_negative==False)\n\n\n    index_negative_and_negative_lte_05 = index_negative & (df_pred_s['negative_exam_for_pe']<=0.5)\n    df_pred_s['negative_exam_for_pe'][index_negative_and_negative_lte_05] = 0.5001\n\n    index_indeterminate_and_indeterminate_lte_05 = index_indeterminate & (df_pred_s['indeterminate']<=0.5)\n    df_pred_s['indeterminate'][index_indeterminate_and_indeterminate_lte_05] = 0.5001\n\n    index_indeterminate_and_negative_gt_05 = index_indeterminate & (df_pred_s['negative_exam_for_pe']>0.5)\n    df_pred_s['negative_exam_for_pe'][index_indeterminate_and_negative_gt_05] = 0.5\n\n    index_negative_and_indeterminate_gt_05 = index_negative & (df_pred_s['indeterminate']>0.5)\n    df_pred_s['indeterminate'][index_negative_and_indeterminate_gt_05] = 0.5\n\n    \n    index_positive_and_negative_gt_05 = index_positive & (df_pred_s['negative_exam_for_pe']>0.5)\n    df_pred_s['negative_exam_for_pe'][index_positive_and_negative_gt_05] = 0.5\n    \n    index_positive_and_indeterminate_gt_05 = index_positive & (df_pred_s['indeterminate']>0.5)\n    df_pred_s['indeterminate'][index_positive_and_indeterminate_gt_05] = 0.5\n    \n    ################################################\n    index_negative_and_rv_lv_ratio_lt_1_gt_05 = (index_positive==False) & (df_pred_s['rv_lv_ratio_lt_1']>0.5)\n    df_pred_s['rv_lv_ratio_lt_1'][index_negative_and_rv_lv_ratio_lt_1_gt_05] = 0.5\n\n    index_negative_and_rv_lv_ratio_gte_1_gt_05 = (index_positive==False) & (df_pred_s['rv_lv_ratio_gte_1']>0.5)\n    df_pred_s['rv_lv_ratio_gte_1'][index_negative_and_rv_lv_ratio_gte_1_gt_05] = 0.5\n\n    index_negative_and_central_pe_gt_05 = (index_positive==False) & (df_pred_s['central_pe']>0.5)\n\n    index_negative_and_rightsided_pe_gt_05 = (index_positive==False) & (df_pred_s['rightsided_pe']>0.5)\n    df_pred_s['rightsided_pe'][index_negative_and_rightsided_pe_gt_05] = 0.5\n\n    index_negative_and_leftsided_pe_gt_05 = (index_positive==False) & (df_pred_s['leftsided_pe']>0.5)\n    df_pred_s['leftsided_pe'][index_negative_and_leftsided_pe_gt_05] = 0.5\n\n    index_negative_and_chronic_pe_gt_05 = (index_positive==False) & (df_pred_s['chronic_pe']>0.5)\n    df_pred_s['chronic_pe'][index_negative_and_chronic_pe_gt_05] = 0.5\n\n    index_negative_and_acute_and_chronic_pe_gt_05 = (index_positive==False) & (df_pred_s['acute_and_chronic_pe']>0.5)\n    df_pred_s['acute_and_chronic_pe'][index_negative_and_acute_and_chronic_pe_gt_05] = 0.5\n\n    ################################################\n    index_positive_and_rv_gte_lv = index_positive & (df_pred_s['rv_lv_ratio_lt_1']<=df_pred_s['rv_lv_ratio_gte_1'])\n    index_positive_and_rv_lt_lv = index_positive & (df_pred_s['rv_lv_ratio_lt_1']>df_pred_s['rv_lv_ratio_gte_1'])\n\n    index_positive_and_rv_gte_lv_and_rv_lv_ratio_gte_1_lte_05 =\\\n        (index_positive_and_rv_gte_lv) & (df_pred_s['rv_lv_ratio_gte_1']<=0.5)\n    df_pred_s['rv_lv_ratio_gte_1'][index_positive_and_rv_gte_lv_and_rv_lv_ratio_gte_1_lte_05] = 0.5001\n\n    index_positive_and_rv_gte_lv_and_rv_lv_ratio_lt_1_gt_05 =\\\n        (index_positive_and_rv_gte_lv) & (df_pred_s['rv_lv_ratio_lt_1']>0.5)\n    df_pred_s['rv_lv_ratio_lt_1'][index_positive_and_rv_gte_lv_and_rv_lv_ratio_lt_1_gt_05] = 0.5\n\n    index_positive_and_rv_lt_lv_and_rv_lv_ratio_lt_1_lte_05 =\\\n        (index_positive_and_rv_lt_lv) & (df_pred_s['rv_lv_ratio_lt_1']<=0.5)\n    df_pred_s['rv_lv_ratio_lt_1'][index_positive_and_rv_lt_lv_and_rv_lv_ratio_lt_1_lte_05] = 0.5001\n\n    index_positive_and_rv_lt_lv_and_rv_lv_ratio_gte_1_gt_05 =\\\n        (index_positive_and_rv_lt_lv) & (df_pred_s['rv_lv_ratio_gte_1']>0.5)\n    df_pred_s['rv_lv_ratio_gte_1'][index_positive_and_rv_lt_lv_and_rv_lv_ratio_gte_1_gt_05] = 0.5\n\n    index_positive_and_central_is_greatest = index_positive & (df_pred_s['central_pe']>=df_pred_s['rightsided_pe']) & (df_pred_s['central_pe']>=df_pred_s['leftsided_pe'])\n    index_positive_and_right_is_greatest = index_positive & (index_positive_and_central_is_greatest==False) & (df_pred_s['rightsided_pe']>=df_pred_s['leftsided_pe'])\n    index_positive_and_left_is_greatest = index_positive & (index_positive_and_central_is_greatest==False) & (index_positive_and_right_is_greatest==False) \n\n\n    index_positive_and_central_is_greatest_and_central_pe_lte_05 = (index_positive_and_central_is_greatest) & (df_pred_s['central_pe']<=0.5)\n    df_pred_s['central_pe'][index_positive_and_central_is_greatest_and_central_pe_lte_05] = 0.5001\n\n    index_positive_and_right_is_greatest_and_rightsided_pe_lte_05 = (index_positive_and_right_is_greatest) & (df_pred_s['rightsided_pe']<=0.5)\n    df_pred_s['rightsided_pe'][index_positive_and_right_is_greatest_and_rightsided_pe_lte_05] = 0.5001\n\n    index_positive_and_left_is_greatest_and_leftsided_pe_lte_05 = (index_positive_and_left_is_greatest) & (df_pred_s['leftsided_pe']<=0.5)\n    df_pred_s['leftsided_pe'][index_positive_and_left_is_greatest_and_leftsided_pe_lte_05] = 0.5001\n\n     # acute_and_chronic_pe and chronic_pe: only one of them can have p > 0.5; neither having p > 0.5 is allowed.\n    index_double_positive = index_positive & (df_pred_s['chronic_pe']>0.5) & (df_pred_s['acute_and_chronic_pe']>0.5)\n\n    index_double_positive_and_chronic_lte_acute_and_chronic = index_double_positive & (df_pred_s['chronic_pe']<=df_pred_s['acute_and_chronic_pe'])\n    df_pred_s['chronic_pe'][index_double_positive_and_chronic_lte_acute_and_chronic] = 0.5\n\n    index_double_positive_and_chronic_gt_acute_and_chronic = index_double_positive & (df_pred_s['chronic_pe']>df_pred_s['acute_and_chronic_pe'])\n    df_pred_s['acute_and_chronic_pe'][index_double_positive_and_chronic_gt_acute_and_chronic] = 0.5\n\n    ################################################\n    df_pred_s['positive'] = 0\n    df_pred_s['positive'][index_positive] = 1\n    df_pred2 = pd.merge(df_pred, df_pred_s[[col_groupby, 'positive']], on=col_groupby, how='left')\n\n    df_agg = df_pred.groupby(col_groupby)['pe_present_on_image'].agg('max').reset_index()\n    df_agg.columns = [col_groupby, 'pe_present_on_image_pred_max']\n    df_pred2 = pd.merge(df_pred2, df_agg, on=col_groupby, how='left')\n    df_pred2['peak'] = df_pred2['pe_present_on_image']==df_pred2['pe_present_on_image_pred_max']\n    # df_tmp = df_s_p[[col_groupby]]\n    # df_tmp['positive'] = True\n\n    index_positive_i = df_pred2['positive']==1\n\n    index_negative_and_pe_present_on_image_gt_05_i = (index_positive_i==False) & (df_pred2['pe_present_on_image']>0.5)\n    df_pred['pe_present_on_image'][index_negative_and_pe_present_on_image_gt_05_i] = 0.5\n\n    index_positive_and_peak_and_pe_present_on_image_lte_05_i = index_positive_i & (df_pred2['peak']) & (df_pred2['pe_present_on_image']<=0.5)\n    df_pred['pe_present_on_image'][index_positive_and_peak_and_pe_present_on_image_lte_05_i] = 0.5001\n   \n    if verbose:\n        print(\"num study\", len(df_pred_s))\n        print(\"num image\", len(df_pred))\n        print(\"split to 3 classes\")\n        print(\" num predicted_as_negative:\", index_negative.sum())\n        print(\" num predicted_as_indeterminate:\", index_indeterminate.sum())\n        print(\" num predicted_as_positive:\", index_positive.sum())\n        print(\"process 3 class conflict\")\n        print(\" num predicted_as_negative and negative<=0.5:\", index_negative_and_negative_lte_05.sum())\n        print(\" num predicted_as_indeterminate and indeterminate<=0.5:\", index_indeterminate_and_indeterminate_lte_05.sum())\n\n        print(\" num predicted_as_indeterminate and negative_exam_for_pe>0.5:\", index_indeterminate_and_negative_gt_05.sum())\n        print(\" num predicted_as_negative and indeterminate>0.5:\", index_negative_and_indeterminate_gt_05.sum())\n        print(\" num predicted_as_positive and negative_exam_for_pe>0.5:\", index_positive_and_negative_gt_05.sum())\n        print(\" num predicted_as_positive and indeterminate>0.5:\", index_positive_and_indeterminate_gt_05.sum())\n      \n        print(\"process negative case\")\n        print(\" num predicted_as_not_positive and rv_lv_ratio_lt_1>0.5:\", index_negative_and_rv_lv_ratio_lt_1_gt_05.sum())\n        print(\" num predicted_as_not_positive and rv_lv_ratio_gte_1>0.5:\", index_negative_and_rv_lv_ratio_gte_1_gt_05.sum())\n        print(\" num predicted_as_not_positive and central_pe>0.5:\", index_negative_and_central_pe_gt_05.sum())\n        print(\" num predicted_as_not_positive and central_pe>0.5:\", index_negative_and_rightsided_pe_gt_05.sum())\n        print(\" num predicted_as_not_positive and leftsided_pe>0.5:\", index_negative_and_leftsided_pe_gt_05.sum())\n        print(\" num predicted_as_not_positive and chronic_pe>0.5:\", index_negative_and_chronic_pe_gt_05.sum())\n        print(\" num predicted_as_not_positive and acute_and_chronic_pe>0.5:\", index_negative_and_acute_and_chronic_pe_gt_05.sum())\n\n        print(\"process positive case\")\n        print(\" num predicted_as_positive and rv_lv_ratio_lt_1<=rv_lv_ratio_gte_1:\", index_positive_and_rv_gte_lv.sum())\n        print(\" num predicted_as_positive and rv_lv_ratio_lt_1>rv_lv_ratio_gte_1:\", index_positive_and_rv_lt_lv.sum())\n        print(\" num predicted_as_positive and (rv_lv_ratio_lt_1<=rv_lv_ratio_gte_1) and (rv_lv_ratio_gte_1<=0.5): \",\n               index_positive_and_rv_gte_lv_and_rv_lv_ratio_gte_1_lte_05.sum())\n        print(\" num predicted_as_positive and (rv_lv_ratio_lt_1<=rv_lv_ratio_gte_1) and rv_lv_ratio_lt_1>0.5: \",\n               index_positive_and_rv_gte_lv_and_rv_lv_ratio_lt_1_gt_05.sum())\n        print(\" num predicted_as_positive and (rv_lv_ratio_lt_1>rv_lv_ratio_gte_1) and rv_lv_ratio_lt_1<=0.5: \",\n               index_positive_and_rv_lt_lv_and_rv_lv_ratio_lt_1_lte_05.sum())\n        print(\" num predicted_as_positive and (rv_lv_ratio_lt_1>rv_lv_ratio_gte_1) and rv_lv_ratio_gte_1>0.5: \",\n               index_positive_and_rv_lt_lv_and_rv_lv_ratio_gte_1_gt_05.sum())\n        print(\" num predicted_as_positive and central is greatest:\", index_positive_and_central_is_greatest.sum())\n        print(\" num predicted_as_positive and right is greatest:\", index_positive_and_right_is_greatest.sum())\n        print(\" num predicted_as_positive and left is greatest:\", index_positive_and_left_is_greatest.sum())\n        print(\" num predicted_as_positive and central is greatest and central_pe<=0.5:\", index_positive_and_central_is_greatest_and_central_pe_lte_05.sum())\n        print(\" num predicted_as_positive and right is greatest and rightsided_pe<=0.5:\", index_positive_and_right_is_greatest_and_rightsided_pe_lte_05.sum())\n        print(\" num predicted_as_positive and left is greatest and leftsided_pe<=0.5:\", index_positive_and_left_is_greatest_and_leftsided_pe_lte_05.sum())\n        print(\" num both chronic_pe and acute_and_chronic_pe is positive:\", index_double_positive.sum())\n        print(\" num both chronic_pe and acute_and_chronic_pe is positive and chronic<=acute_and_chronic:\", index_double_positive_and_chronic_lte_acute_and_chronic.sum())\n        print(\" num both chronic_pe and acute_and_chronic_pe is positive and chronic>acute_and_chronic:\", index_double_positive_and_chronic_gt_acute_and_chronic.sum())\n\n        print(\"process image level\")\n        print(\" num img of predicted_as_positive:\", index_positive_i.sum())\n        print(\" num img of predicted_as_negative:\", (index_positive_i==0).sum())\n        print(\" num img of peak:\", df_pred2['peak'].sum())\n        print(\" num img of predicted_as_negative and pe_present_on_image>0.5:\", index_negative_and_pe_present_on_image_gt_05_i.sum())\n        print(\" num img of predicted_as_positive and peak and pe_present_on_image<=0.5:\", index_positive_and_peak_and_pe_present_on_image_lte_05_i.sum())\n\n    return df_pred_s, df_pred\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pred_study_const, df_pred_image_const = solve_conflict(df_pred_study, df_pred_image)\ndf_pred_study_const.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub_pred = copy.deepcopy(df_pred_image_const[[col_index, col_targets[-1]]])\ndf_sub_pred.columns = ['id', 'label']\nfor i, col in enumerate(col_targets[:-1]):\n    df_tmp = df_pred_study_const[[col_groupby, col]]\n    df_tmp.columns = ['id', 'label']\n    df_tmp['id'] = df_tmp['id'] + '_{}'.format(col)\n    df_sub_pred = pd.concat([df_sub_pred, df_tmp])\ndf_sub_pred = df_sub_pred.reset_index(drop=True)\nprint(df_sub_pred.shape)\ndf_sub_pred.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub = pd.read_csv(df_sub_path)\nprint(df_sub.shape)\ndf_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub = pd.merge(df_sub[['id']], df_sub_pred, on='id', how='left')\n# df_sub = df_sub.fillna(0.5)\nprint(df_sub.shape)\ndf_sub.head(30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# nan埋め\nmean_targets = [\n    0.674681,\n    0.021569,\n    0.040115,\n    0.019920,\n    0.055090,\n    0.212117,\n    0.257590,\n    0.129139,\n    0.174612,\n    0.289885,\n]\ndf_sub_mean = copy.deepcopy(df_test_full[[col_index]])\ndf_sub_mean.columns = ['id']\ndf_sub_mean['label'] = mean_targets[-1]\nfor i, col in enumerate(col_targets[:-1]):\n    df_tmp = df_test_full_study[[col_groupby]]\n    df_tmp.columns = ['id']\n    df_tmp['label'] = mean_targets[i]\n    df_tmp['id'] = df_tmp['id'] + '_{}'.format(col)\n    df_sub_mean = pd.concat([df_sub_mean, df_tmp])\ndf_sub_mean = df_sub_mean.reset_index(drop=True)\nprint(df_sub_mean.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub['label'][pd.isna(df_sub['label'])] = pd.merge(df_sub[['id']], df_sub_mean, on='id', how='left')['label'][pd.isna(df_sub['label'])]\ndf_sub.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_consistency2(df_exam, df_image, test):\n    \n    '''\n    Checks label consistency and returns the errors\n    \n    Args:\n    sub   = submission dataframe (pandas)\n    test  = test.csv dataframe (pandas)\n    '''\n\n    \n    # MERGER\n    df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')\n    ids    = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']\n    labels = [c for c in df.columns if c not in ids]\n    df = df[ids + labels]\n    \n    # SPLIT NEGATIVE AND POSITIVE EXAMS\n    df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID']).pe_present_on_image.max())\n    df_pos = df.loc[df.positive_images_in_exam >  0.5]\n    df_neg = df.loc[df.positive_images_in_exam <= 0.5]\n    \n    # CHECKING CONSISTENCY OF POSITIVE EXAM LABELS\n    rule1a = df_pos.loc[((df_pos.rv_lv_ratio_lt_1  >  0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 >  0.5)) | \n                        ((df_pos.rv_lv_ratio_lt_1  <= 0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 <= 0.5))].reset_index(drop = True)\n    rule1a['broken_rule'] = '1a'\n    rule1b = df_pos.loc[(df_pos.central_pe    <= 0.5) & \n                        (df_pos.rightsided_pe <= 0.5) & \n                        (df_pos.leftsided_pe  <= 0.5)].reset_index(drop = True)\n    rule1b['broken_rule'] = '1b'\n    rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5) & \n                        (df_pos.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule1c['broken_rule'] = '1c'\n    rule1d = df_pos.loc[(df_pos.indeterminate        > 0.5) | \n                        (df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)\n    rule1d['broken_rule'] = '1d'\n\n    # CHECKING CONSISTENCY OF NEGATIVE EXAM LABELS\n    rule2a = df_neg.loc[((df_neg.indeterminate        >  0.5)  & \n                         (df_neg.negative_exam_for_pe >  0.5)) | \n                        ((df_neg.indeterminate        <= 0.5)  & \n                         (df_neg.negative_exam_for_pe <= 0.5))].reset_index(drop = True)\n    rule2a['broken_rule'] = '2a'\n    rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1     > 0.5) | \n                        (df_neg.rv_lv_ratio_gte_1    > 0.5) |\n                        (df_neg.central_pe           > 0.5) | \n                        (df_neg.rightsided_pe        > 0.5) | \n                        (df_neg.leftsided_pe         > 0.5) |\n                        (df_neg.acute_and_chronic_pe > 0.5) | \n                        (df_neg.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule2b['broken_rule'] = '2b'\n    \n    # MERGING INCONSISTENT PREDICTIONS\n    errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)\n    \n    # OUTPUT\n    print('Found', len(errors), 'inconsistent predictions')\n    return errors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"error = check_consistency2(df_pred_study_const, df_pred_image_const, df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if len(error)==0:\n    df_sub.to_csv('submission.csv', index=None)\nelse:\n    print(\"error!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"3989*3/3600*2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}