{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip uninstall -y tensorflow\n!pip install tensorflow-gpu==1.15.3\n!pip install ../input/keras-mod2/Keras-2.3.1\n!pip install keras-resnet==0.1.0\n!pip install keras-retinanet==0.5.1\n!nvidia-smi","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport gzip\nimport pickle\nimport os\nimport glob\nimport time\nimport cv2\nimport datetime\nimport pandas as pd\nfrom sklearn.metrics import fbeta_score\nfrom sklearn.model_selection import KFold, train_test_split\nfrom collections import Counter, defaultdict\nfrom sklearn.metrics import accuracy_score, roc_auc_score, log_loss\nimport random\nimport shutil\nimport operator\nfrom PIL import Image\nimport platform\nimport json\nimport base64\nimport typing as t\nimport zlib\nimport pydicom\nimport re\nfrom tqdm import tqdm\n\nROOT_PATH = './'\nINPUT_PATH = '../input/vinbigdata-chest-xray-abnormalities-detection/'\nOUTPUT_PATH = ROOT_PATH + 'modified_data_folder/'\nif not os.path.isdir(OUTPUT_PATH):\n    os.mkdir(OUTPUT_PATH)\nMODELS_PATH = ROOT_PATH + 'models_inference/'\nif not os.path.isdir(MODELS_PATH):\n    os.mkdir(MODELS_PATH)\nCACHE_PATH = ROOT_PATH + 'cache_folder/'\nif not os.path.isdir(CACHE_PATH):\n    os.mkdir(CACHE_PATH)\nFEATURES_PATH = ROOT_PATH + 'features_folder/'\nif not os.path.isdir(FEATURES_PATH):\n    os.mkdir(FEATURES_PATH)\nSUBM_PATH = ROOT_PATH + 'subm_folder/'\n\n\ndef save_in_file(arr, file_name):\n    pickle.dump(arr, gzip.open(file_name, 'wb+', compresslevel=3), protocol=4)\n\n\ndef load_from_file(file_name):\n    return pickle.load(gzip.open(file_name, 'rb'))\n\n\ndef save_in_file_fast(arr, file_name):\n    pickle.dump(arr, open(file_name, 'wb'), protocol=4)\n\n\ndef load_from_file_fast(file_name):\n    return pickle.load(open(file_name, 'rb'))\n\n\ndef get_classes_array():\n    train = pd.read_csv(INPUT_PATH + 'train.csv')\n    res = dict()\n    for index, row in train.iterrows():\n        class_id = row['class_id']\n        class_name = row['class_name']\n        if class_id not in res:\n            res[class_id] = class_name\n        else:\n            if res[class_id] != class_name:\n                print('Error')\n    CLASSES = []\n    for i in range(15):\n        CLASSES.append(res[i])\n    return CLASSES\n\n\ndef read_single_image(path):\n    try:\n        img = cv2.imread(path, cv2.IMREAD_ANYDEPTH)\n    except:\n        print('Fail')\n        return np.zeros((512, 512, 3), dtype=np.uint8)\n\n    if len(img.shape) == 2:\n        img = np.stack([img, img, img], axis=-1)\n\n    if img.shape[2] == 2:\n        img = img[:, :, :1]\n\n    if img.shape[2] == 1:\n        img = np.concatenate((img, img, img), axis=2)\n\n    if img.shape[2] > 3:\n        img = img[:, :, :3]\n\n    return img\n\n\ndef get_retinanet_predictions_for_files(model_path, files, out_dir, backbone, min_width, max_width, lvl_labels):\n    from keras_retinanet.utils.image import preprocess_image, resize_image\n    from keras_retinanet import models\n\n    show_debug_images = False\n    show_mirror_predictions = False\n\n    model = models.load_model(model_path, backbone_name=backbone)\n    print('Proc {} files...'.format(len(files)))\n    result_data = dict()\n    for f in files:\n        id = os.path.basename(f)[:-4]\n\n        cache_path = out_dir + id + '.pkl'\n        if os.path.isfile(cache_path):\n           continue\n\n        # try:\n        image = read_single_image(f)\n\n        if show_debug_images:\n            # copy to draw on\n            draw = image.copy()\n            draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)\n\n        # preprocess image for network\n        image = preprocess_image(image)\n        image, scale = resize_image(image, min_side=min_width, max_side=max_width)\n\n        # Add mirror\n        image = np.stack((image, image[:, ::-1, :]), axis=0)\n\n        # process image\n        start = time.time()\n        print('ID: {} Image shape: {} Scale: {}'.format(id, image.shape, scale))\n        boxes, scores, labels = model.predict_on_batch(image)\n        print('Detections shape: {} {} {}'.format(boxes.shape, scores.shape, labels.shape))\n        print(\"Processing time: {:.2f} sec\".format(time.time() - start))\n\n        if show_debug_images:\n            if show_mirror_predictions:\n                draw = draw[:, ::-1, :]\n            boxes_init = boxes.copy()\n            boxes_init /= scale\n\n        boxes[:, :, 0] /= image.shape[2]\n        boxes[:, :, 2] /= image.shape[2]\n        boxes[:, :, 1] /= image.shape[1]\n        boxes[:, :, 3] /= image.shape[1]\n\n        if show_debug_images:\n            if show_mirror_predictions:\n                show_image_debug(lvl_labels, draw.astype(np.uint8), boxes_init[1:], scores[1:], labels[1:])\n            else:\n                show_image_debug(lvl_labels, draw.astype(np.uint8), boxes_init[:1], scores[:1], labels[:1])\n\n        # save_in_file_fast((boxes, scores, labels), cache_path)\n        result_data[id] = (boxes, scores, labels)\n    cache_path = out_dir + '_result.pkl'\n    save_in_file_fast(result_data, cache_path)\n\n\ndef get_retinanet_preds_for_tst(files, model_path, backbone, reverse, min_width, max_width, out_path_prefix, labels):\n    if reverse is True:\n        files = files[::-1]\n    get_retinanet_predictions_for_files(model_path, files, out_path_prefix + '_test', backbone, min_width, max_width, labels)\n\n\ndef get_retinanet_preds_for_valid(files, model_path, backbone, reverse, min_width, max_width, out_path_prefix, labels):\n    if reverse is True:\n        files = files[::-1]\n    get_retinanet_predictions_for_files(model_path, files, out_path_prefix + '_valid', backbone, min_width, max_width, labels)\n\n\nif __name__ == '__main__':\n\n    mp = '../input/chest-xray-abnormalities-detection-models/retinanet_resnet101_sqr_removed_rads/'\n    model_list = [\n        mp + 'resnet101_fold_0_0.1817_05_iou_0.3_converted.h5',\n        mp + 'resnet101_fold_1_0.2072_19_iou_0.3_converted.h5',\n        mp + 'resnet101_fold_2_0.1938_03_iou_0.3_converted.h5',\n        mp + 'resnet101_fold_3_0.1884_07_iou_0.3_converted.h5',\n        mp + 'resnet101_fold_4_0.2227_05_iou_0.3_converted.h5',\n    ]\n\n    calc_fold = [3]\n\n    for fold in range(len(model_list)):\n        if fold not in calc_fold:\n            continue\n        print('Go fold: {}'.format(fold))\n        reverse = False\n        model_path = model_list[fold]\n        min_width = 1024\n        max_width = 1024\n        backbone = 'resnet101'\n        out_path_prefix = OUTPUT_PATH + os.path.basename(model_path)[:-13]\n        labels = get_classes_array()\n        test_files = sorted(glob.glob('../input/chest-xray-test-diff-sizes/test_png_div_2/*.png'))\n        print('Files found: {}'.format(len(test_files)))\n        get_retinanet_preds_for_tst(test_files, model_path, backbone, reverse, min_width, max_width, out_path_prefix, labels)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}