{"cells":[{"metadata":{},"cell_type":"markdown","source":"Okay, to start off, I'm not a computer vision specialist, as I worked more with text.\n\nBUT, I'll share and try to mix everything I find that seems cool or good or full of potential\n\nHope you enjoy ❤"},{"metadata":{},"cell_type":"markdown","source":"# Shoutouts:\nMany thanks to the authors of these kernels (a.k.a. I'm basing this on these guys):\n* Model: https://www.kaggle.com/corochann/vinbigdata-detectron2-train/data#Dataset-preparation\n* Data processing and resizing: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-original-ratio / https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n* Data reading: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n* Prediction Kernel: https://www.kaggle.com/corochann/vinbigdata-detectron2-prediction"},{"metadata":{},"cell_type":"markdown","source":"# Imports"},{"metadata":{},"cell_type":"markdown","source":"## Installations"},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvcc --version","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip uninstall detectron2 -y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.7/index.html","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"######################################## Python/General #######################################\nimport os\nfrom PIL import Image # it's image related, but it's kinda default to python, so it stays here\nfrom tqdm import tqdm\nimport pickle\nimport pathlib\nfrom pathlib import Path\nimport typing\nfrom typing import Optional, Any, Union, Dict, List\nfrom tqdm import tqdm\nimport yaml\nimport contextlib\nimport copy\nimport io\nimport itertools\nimport json\nimport logging\nfrom collections import OrderedDict\nfrom tabulate import tabulate\nimport dataclasses\nfrom dataclasses import dataclass\nfrom typing import Dict\nimport argparse\nimport random\nimport sys\nfrom distutils.util import strtobool\nimport math\n\n######################################## Data science #########################################\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\n\n%matplotlib inline\n\n######################################## Image related ########################################\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\n\n# Dataset\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    print(dirname)  \n    \n######################################## Model related ########################################    \nimport torch\nimport torch.nn as nn\nfrom detectron2.structures import BoxMode\nimport detectron2.data.transforms as T\nfrom detectron2.data import detection_utils as utils\nfrom detectron2 import model_zoo\nfrom detectron2.config import get_cfg\nfrom detectron2.data import DatasetCatalog, MetadataCatalog\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer, launch\nfrom detectron2.evaluation import COCOEvaluator\nfrom detectron2.structures import BoxMode\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.utils.visualizer import ColorMode, Visualizer\n\nprint(torch.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n        \n    # Depending on this value, X-ray may look inverted - fix that\n    if fix_monochrome and dicom.PhotometricInterpretation == 'MONOCHROME1\"':\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = read_xray('../input/vinbigdata-chest-xray-abnormalities-detection/train/000434271f63a053c4128a0ba6352c7f.dicom')\nplt.figure(figsize = (12, 12))\nplt.imshow(img, 'gray')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Resizing"},{"metadata":{"trusted":true},"cell_type":"code","source":"#def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n#    im = Image.fromarray(array)\n    \n#    if keep_ratio:\n#        im.thumbnail((size, size), resample)\n#    else:\n#        im = im.resize((size, size), resample)\n        \n#    return im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#image_id = []\n#dim0 = []\n#dim1 = []\n\n#for split in ['train', 'test']:\n#    load_dir = f'../input/vinbigdata-chest-xray-abnormalities-detection/{split}/'\n#    save_dir = f'/kaggle/tmp/{split}/'\n    \n#    os.makedirs(save_dir, exist_ok=True)\n\n#    for file in tqdm(os.listdir(load_dir)):\n#        # set keep_ratio=True to have originicar aspect ratio\n#        xray = read_xray(load_dir + file)\n#        im = resize(xray, size=512, keep_ratio=True)\n#        im.save(save_dir + file.replace('dicom', 'png'))\n        \n#        if split == 'train':\n#           image_id.append(file.replace('.dicom', ''))\n#            dim0.append(xray.shape[0])\n#            dim1.append(xray.shape[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference methods"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_vinbigdata_dicts(imgdir, train_df, train_data_type='original', \n                         use_cache=True, debug=True, \n                         np_ndarr_target_indices=None, use_class14=False):\n    \n    debug_str = f'_debug{int(debug)}'\n    \n    train_data_type_str = f'_{train_data_type}'\n    \n    class_14_str = f'_14_class{int(use_class_14)}'\n    \n    cache_path = Path('.') / f'dataset_dicts_cache{train_data_type_str}{class_14_str}{debug_str}.pkl'\n    \n    if not use_cache or not cache_path.exists():\n        print('Creating data...')\n        train_meta = pd.read_csv(imgdir / 'train_meta.csv')\n        \n        if debug:\n            train_df = train_df.iloc[:500]\n        \n        image_id = train_df.loc[0, 'image_id']\n        image_path = str(imgdir / 'train' / f'{image_id}.png')\n        image = cv2.imread(image_path)\n        img_height, img_width, ch = image.shape\n        \n        print(f'image shape: {image.shape}')\n        \n        dataset_dicts = []\n        \n        for index, train_meta_row in tqdm(train_meta.iterrows(), total=len(traqin_meta)):\n            \n            image_id, height, width = train_meta_row.values\n            filename = str(imgdir / 'train' / f'{image_id}.png')\n            \n            record = {}\n            \n            record['file_name'] = filename\n            record['image_id']  = image_id\n            record['height']    = img_height\n            record['width']     = img_width\n            \n            objects = []\n\n            for index2, row in train_df.query('image_id == @image_id').iterrows():\n                class_id = row['class_id']\n                if class_id == 14:\n                    if use_class_14:\n                        bbox_resized = [0, 0, img_width, img_height]\n                        \n                        obj = {}\n                        \n                        obj['bbox']        = bbox_resized\n                        obj['bbox_mode']   = BoxMode.XYXY_ABS\n                        obj['category_id'] = class_id\n                        \n                        objs.append(obj)\n                    else:\n                        pass\n                    \n                else:\n                    h_ratio = img_height / heigth\n                    w_ratio = img_width / width\n                    bbox_resized = [float(row['x_min']) * w_ratio,\n                                    float(row['y_min']) * h_ratio,\n                                    float(row['x_max']) * w_ratio,\n                                    float(row['y_max']) * h_ratio]\n                    \n                    obj = {}\n                    \n                    obj['bbox'] = bbox_resized\n                    obj['bbox_mode'] = BoxMode.XYXY_ABS\n                    obj['category_id'] = class_id\n                    \n                    objs.append(obj)\n                    \n            record['annotations'] = objs\n            dataset_dict.append(record)\n            \n        with open(cache_path, mode='wb') as f:\n            pickle.dump(dataset_dicts, f)\n            \n    print(f'Load from cache {cache_path}')\n    with open(cache_path, mode='rb') as f:\n        dataset_dicts = pickle.load(f)\n    \n    if target_indices is not None:\n        dataset_dicts = [dataset_dicts[i] for i in target_indices]\n    return dataset_dicts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_vinbigdata_dicts_test(Path, test_df, use_cache=True, debug=True):\n    debug_str = f'_debug{int(debug)}'\n    cache_path = pathlib.Path('.') / f'dataset_dicts_cache_test{debug_str}.pkl'\n    \n    if not use_cache or not cache_path.exists():\n        print(\"Creating data...\")\n        \n        if debug:\n            test_df = test_df.iloc[:500]\n            \n        image_id = test_df.loc[0, 'image_id']\n        image_path = str(imgdir / 'test' / f'{image_id}.png')\n        image = cv2.imread(image_path)\n        img_height, img_width, ch = image.shape\n        \n        print(f'image shape: {image.shape}')\n        \n        dataset_dicts = []\n        \n        for index, test_meta_row in tqdm(test_meta.iterrows(), total = len(test_meta)):\n            image_id, img_heigth, img_width = test_meta_row.values\n            filename = str(imgdir / 'test' / f'{image_id}.png')\n            \n            record = {}\n            \n            record['file_name'] = filename\n            record['image_id']  = image_id\n            record['height']    = img_heigth\n            record['width']     = img_width\n            \n            dataset_dicts.append(record)\n            \n        with open(cache_path, mode=\"wb\") as f:\n            pickle.dump(dataset_dicts, f)\n\n    print(f\"Load from cache {cache_path}\")\n    \n    with open(cache_path, mode=\"rb\") as f:\n        dataset_dicts = pickle.load(f)\n        \n    return dataset_dicts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_yaml(filepath: Union[str, Path], content: Any, width: int=120):\n    with open(filepath, \"w\") as f:\n        yaml.dump(content, f, width=width)\n\n\ndef load_yaml(filepath: Union[str, Path]) -> Any:\n    with open(filepath, \"r\") as f:\n        content = yaml.full_load(f)\n        \n    return content","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_pred(labels: np.ndarray, boxes: np.ndarray, scores: np.ndarray) -> str:\n    pred_strings = []\n    for label, score, bbox in zip(labels, scores, boxes):\n        xmin, ymin, xmax, ymax = bbox.astype(np.int64)\n        pred_strings.append(f\"{label} {score} {xmin} {ymin} {xmax} {ymax}\")\n        \n    return \" \".join(pred_strings)\n\n\ndef predict_batch(predictor: DefaultPredictor, im_list) -> typing.List:\n    with torch.no_grad():  # https://github.com/sphinx-doc/sphinx/issues/4258\n        inputs_list = []\n        for original_image in im_list:\n            # Apply pre-processing to image.\n            if predictor.input_format == \"RGB\":\n                # whether the model expects BGR inputs or RGB\n                original_image = original_image[:, :, ::-1]\n            height, width = original_image.shape[:2]\n            # Do not apply original augmentation, which is resize.\n            # image = predictor.aug.get_transform(original_image).apply_image(original_image)\n            image = original_image\n            image = torch.as_tensor(image.astype(\"float32\").transpose(2, 0, 1))\n            inputs = {\"image\": image, \"height\": height, \"width\": width}\n            inputs_list.append(inputs)\n        predictions = predictor.model(inputs_list)\n        return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"region_classes = [\n    'Aortic enlargement',\n    'Atelectasis',\n    'Calcification',\n    'Cardiomegaly',\n    'Consolidation',\n    'ILD',\n    'Infiltration',\n    'Lung opacity',\n    'Nodule/Mass',\n    'Other lesion'\n    'Pleural effusion',\n    'Pleural thickening',\n    'Pneumothorax',\n    'Pulmonary fibrosis'\n]\n\ncategory_name_to_id = {class_name: index for index, class_name in enumerate(region_classes)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    # General\n    debug = True\n    outdir = 'results/det'\n    \n    # Data config\n    #vinbigdata-chest-xray-resized-png-256x256\n    imgdir_name = 'vinbigdata'\n    split_mode = 'all_train'  # all_train or valid20\n    seed = 111\n    train_data_type = 'original'  # original or wbf\n    use_class14 = False\n        \n    # Training config\n    iter = 10000\n    ims_per_batch = 2  # images per batch, this corresponds to \"total batch size\"\n    num_workers = 4\n    lr_scheduler_name = 'WarmupMultiStepLR'  # WarmupMultiStepLR (default) or WarmupCosineLR\n    base_lr = 0.00025\n    roi_batch_size_per_image = 512\n    eval_period = 10000\n    aug_kwargs = dataclasses.field(default_factory=lambda: {})\n\n    def update(self, param_dict) -> 'Config':\n        # Overwrite by `param_dict`\n        for key, value in param_dict.items():\n            if not hasattr(self, key):\n                raise ValueError(f'[ERROR] Unexpected key for config = {key}')\n            setattr(self, key, value)\n        return self","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inputdir = Path('/kaggle/input')\ntraineddir = inputdir / 'vinbigdata-r50fpn3x-512px'\n\nconfigs = Config().update(load_yaml(str(traineddir/'flags.yaml'))) # TODO: change to configs.yaml when there's time to gen \nprint('configs', configs)\ndebug = configs.debug\noutdir = Path(configs.outdir)\nos.makedirs(str(outdir), exist_ok=True)\n\n# --- Read data ---\ndatadir = inputdir / 'vinbigdata-chest-xray-abnormalities-detection'\nif configs.imgdir_name == 'vinbigdata2':\n    #vinbigdata\n    imgdir = inputdir/ 'vinbigdata-chest-xray-resized-png-256x256'\nelse:\n    imgdir = inputdir / configs.imgdir_name\n\n# Read in the data CSV files\n# train = pd.read_csv(datadir / \"train.csv\")\ntest_meta = pd.read_csv(inputdir / 'vinbigdata-testmeta' / 'test_meta.csv')\nsample_submission = pd.read_csv(datadir / 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg = get_cfg()\noriginal_output_dir = cfg.OUTPUT_DIR\ncfg.OUTPUT_DIR = str(outdir)\nprint(f'cfg.OUTPUT_DIR {original_output_dir} -> {cfg.OUTPUT_DIR}')\n\ncfg.merge_from_file(model_zoo.get_config_file('COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml'))\ncfg.DATASETS.TRAIN = ('vinbigdata_train',)\ncfg.DATASETS.TEST = ()\ncfg.DATALOADER.NUM_WORKERS = 2\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url('COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml')\ncfg.SOLVER.IMS_PER_BATCH = 2\ncfg.SOLVER.BASE_LR = configs.base_lr  # pick a good LR\ncfg.SOLVER.MAX_ITER = configs.iter\ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = configs.roi_batch_size_per_image\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = len(region_classes)\n\n### --- Inference & Evaluation ---\n# Inference should use the config with parameters that are used in training\n# cfg now already contains everything we've set previously. We changed it a little bit for inference:\n# path to the model we just trained\ncfg.MODEL.WEIGHTS = str(traineddir/'model_final.pth')\nprint('Original thresh', cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST)  # 0.05\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.13  # set a custom testing threshold\nprint('Changed  thresh', cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST)\npredictor = DefaultPredictor(cfg)\n\n#DatasetCatalog.register(\n#    'vinbigdata_test', lambda: get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)\n#)\nMetadataCatalog.get('vinbigdata_test').set(thing_classes=region_classes)\nmetadata = MetadataCatalog.get('vinbigdata_test')\ndataset_dicts = get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)\n\nif debug:\n    dataset_dicts = dataset_dicts[:100]\n\nresults_list = []\nindex = 0\nbatch_size = 4\n\nfor i in tqdm(range(math.ceil(len(dataset_dicts) / batch_size))):\n    inds = list(range(batch_size * i, min(batch_size * (i + 1), len(dataset_dicts))))\n    dataset_dicts_batch = [dataset_dicts[i] for i in inds]\n    im_list = [cv2.imread(d['file_name']) for d in dataset_dicts_batch]\n    outputs_list = predict_batch(predictor, im_list)\n\n    for im, outputs, d in zip(im_list, outputs_list, dataset_dicts_batch):\n        resized_height, resized_width, ch = im.shape\n        # outputs = predictor(im)\n        if index < 5:\n            # format is documented at https://detectron2.readthedocs.io/tutorials/models.html#model-output-format\n            v = Visualizer(\n                im[:, :, ::-1],\n                metadata=metadata,\n                scale=0.5,\n                instance_mode=ColorMode.IMAGE_BW\n                # remove the colors of unsegmented pixels. This option is only available for segmentation models\n            )\n            out = v.draw_instance_predictions(outputs['instances'].to('cpu'))\n            # cv2_imshow(out.get_image()[:, :, ::-1])\n            cv2.imwrite(str(outdir / f'pred_{index}.jpg'), out.get_image()[:, :, ::-1])\n\n        image_id, dim0, dim1 = test_meta.iloc[index].values\n\n        instances = outputs['instances']\n        if len(instances) == 0:\n            # No finding, let's set 14 1 0 0 1 1x.\n            result = {'image_id': image_id, 'PredictionString': '14 1.0 0 0 1 1'}\n        else:\n            # Find some bbox...\n            # print(f\"index={index}, find {len(instances)} bbox.\")\n            fields: Dict[str, Any] = instances.get_fields()\n            pred_classes = fields['pred_classes']  # (n_boxes,)\n            pred_scores = fields['scores']\n            # shape (n_boxes, 4). (xmin, ymin, xmax, ymax)\n            pred_boxes = fields['pred_boxes'].tensor\n\n            h_ratio = dim0 / resized_height\n            w_ratio = dim1 / resized_width\n            pred_boxes[:, [0, 2]] *= w_ratio\n            pred_boxes[:, [1, 3]] *= h_ratio\n\n            pred_classes_array = pred_classes.cpu().numpy()\n            pred_boxes_array = pred_boxes.cpu().numpy()\n            pred_scores_array = pred_scores.cpu().numpy()\n\n            result = {\n                'image_id': image_id,\n                'PredictionString': format_pred(\n                    pred_classes_array, pred_boxes_array, pred_scores_array\n                ),\n            }\n        results_list.append(result)\n        index += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_det = pd.DataFrame(results_list, columns=['image_id', 'PredictionString'])\nsubmission_det.to_csv(outdir/\"submission.csv\", index=False)\nsubmission_det","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}