{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"},{"sourceId":1664376,"sourceType":"datasetVersion","datasetId":985270},{"sourceId":4690722,"sourceType":"datasetVersion","datasetId":2708760},{"sourceId":4718233,"sourceType":"datasetVersion","datasetId":2708647}],"dockerImageVersionId":30356,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#from other collab\n\n!pip install -q /kaggle/input/rsna-bcd-whl-ds/python_gdcm-3.0.20-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install -q /kaggle/input/rsna-bcd-whl-ds/pylibjpeg-1.4.0-py3-none-any.whl\n!pip install -q /kaggle/input/rsna-bcd-whl-ds/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!cp -r /kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle /tmp/ && pip install -q /tmp/efficientnet_kaggle","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:49:18.488634Z","iopub.execute_input":"2023-11-27T07:49:18.488955Z","iopub.status.idle":"2023-11-27T07:51:28.189791Z","shell.execute_reply.started":"2023-11-27T07:49:18.488882Z","shell.execute_reply":"2023-11-27T07:51:28.188624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'  # to avoid too many logging messages\nimport pandas as pd, numpy as np, random, shutil\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nimport sklearn\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nimport yaml\n\nfrom IPython import display as ipd\nfrom glob import glob\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:51:40.558314Z","iopub.execute_input":"2023-11-27T07:51:40.558954Z","iopub.status.idle":"2023-11-27T07:51:40.565944Z","shell.execute_reply.started":"2023-11-27T07:51:40.558915Z","shell.execute_reply":"2023-11-27T07:51:40.564959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('np:', np.__version__)\nprint('pd:', pd.__version__)\nprint('sklearn:', sklearn.__version__)\nprint('tf:',tf.__version__)\nprint('tfa:', tfa.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:51:44.03392Z","iopub.execute_input":"2023-11-27T07:51:44.034269Z","iopub.status.idle":"2023-11-27T07:51:44.040435Z","shell.execute_reply.started":"2023-11-27T07:51:44.03424Z","shell.execute_reply":"2023-11-27T07:51:44.039399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    debug         = False\n    \n    # use verbose=0 for silent, vebose=1 for interactive,\n    verbose      = 1 if debug else 0\n    \n    # device\n    device = \"GPU\" # or \"GPU\"\n\n    # seed for data-split, layer init, augs\n    seed = 42\n    \n    # which folds of ckpt to use\n    num_folds = 1\n\n    # dicom to png size\n    resize_dim = 2048\n    aspect_ratio = True\n    \n    # size of training image\n    img_size = [2048, 1024]\n\n    # augmentation\n    augment   = True\n\n    # scale-shift-rotate-shear\n    transform = False\n    fill_mode = 'constant'\n    rot    = 2.0\n    shr    = 2.0\n    hzoom  = 50.0\n    wzoom  = 50.0\n    hshift = 10.0\n    wshift = 10.0\n\n    # flip\n    hflip = False\n    vflip = False\n\n    # clip\n    clip = False\n\n    # lr-scheduler\n    scheduler   = 'exp' # cosine\n\n    # dropout\n    drop_prob   = 0.0\n    drop_cnt    = 10\n    drop_size   = 0.08\n\n    # pixel-augment\n    pixel_aug = False\n    sat  = [0.7, 1.3]\n    cont = [0.8, 1.2]\n    bri  = 0.15\n    hue  = 0.05\n\n    # threshold\n    thr = 0.70\n    \n    # test-time augs\n    flip_model = False\n    tta = 1\n    \n    # target column\n    target_col  = ['cancer']","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:51:46.878716Z","iopub.execute_input":"2023-11-27T07:51:46.879463Z","iopub.status.idle":"2023-11-27T07:51:46.888936Z","shell.execute_reply.started":"2023-11-27T07:51:46.87943Z","shell.execute_reply":"2023-11-27T07:51:46.887928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n#     os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)\n    tf.random.set_seed(SEED)\n    print('seeding done!!!')\nseeding(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:51:55.306961Z","iopub.execute_input":"2023-11-27T07:51:55.307729Z","iopub.status.idle":"2023-11-27T07:51:55.315001Z","shell.execute_reply.started":"2023-11-27T07:51:55.307688Z","shell.execute_reply":"2023-11-27T07:51:55.313561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if \"TPU\" in CFG.device:\n    tpu = 'local' if CFG.device=='TPU-1VM' else None\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=tpu)\n        strategy = tf.distribute.TPUStrategy(tpu)\n    except:\n        CFG.device = \"GPU\"\n        \nif CFG.device == \"GPU\"  or CFG.device==\"CPU\":\n    ngpu = len(tf.config.experimental.list_physical_devices('GPU'))\n    if ngpu>1:\n        print(\"Using multi GPU\")\n        strategy = tf.distribute.MirroredStrategy()\n    elif ngpu==1:\n        print(\"Using single GPU\")\n        strategy = tf.distribute.get_strategy()\n    else:\n        print(\"Using CPU\")\n        strategy = tf.distribute.get_strategy()\n        CFG.device = \"CPU\"\n\nif CFG.device == \"GPU\":\n    print(\"Num GPUs Available: \", ngpu)\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:51:59.070428Z","iopub.execute_input":"2023-11-27T07:51:59.070868Z","iopub.status.idle":"2023-11-27T07:51:59.263853Z","shell.execute_reply.started":"2023-11-27T07:51:59.07083Z","shell.execute_reply":"2023-11-27T07:51:59.262488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/rsna-breast-cancer-detection'\nIMG_DIR = '/tmp/dataset/rsna-bcd'","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:52:06.352898Z","iopub.execute_input":"2023-11-27T07:52:06.353317Z","iopub.status.idle":"2023-11-27T07:52:06.358429Z","shell.execute_reply.started":"2023-11-27T07:52:06.353282Z","shell.execute_reply":"2023-11-27T07:52:06.357328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train\ndf = pd.read_csv(f'{BASE_PATH}/train.csv')\ndf['dicom_path'] = f'{BASE_PATH}/train_images'\\\n                    + '/' + df.patient_id.astype(str)\\\n                    + '/' + df.image_id.astype(str)\\\n                    + '.dcm'\ndf['image_path'] = df.dicom_path.str.replace('.dcm','.png').str.replace(BASE_PATH, IMG_DIR)\nprint('Train:')\ndisplay(df.head(2))\n\n# test\ntest_df = pd.read_csv(f'{BASE_PATH}/test.csv')\ntest_df['dicom_path'] = f'{BASE_PATH}/test_images'\\\n                    + '/' + test_df.patient_id.astype(str)\\\n                    + '/' + test_df.image_id.astype(str)\\\n                    + '.dcm'\ntest_df['image_path'] = test_df.dicom_path.str.replace('.dcm','.png').str.replace(BASE_PATH, IMG_DIR)\nprint('\\nTest:')\ndisplay(test_df.head(2))","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:52:11.26718Z","iopub.execute_input":"2023-11-27T07:52:11.267645Z","iopub.status.idle":"2023-11-27T07:52:11.801637Z","shell.execute_reply.started":"2023-11-27T07:52:11.267602Z","shell.execute_reply":"2023-11-27T07:52:11.80069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.io.gfile.exists(df.dicom_path.iloc[0]), tf.io.gfile.exists(test_df.dicom_path.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:52:22.103666Z","iopub.execute_input":"2023-11-27T07:52:22.104052Z","iopub.status.idle":"2023-11-27T07:52:22.124885Z","shell.execute_reply.started":"2023-11-27T07:52:22.104015Z","shell.execute_reply":"2023-11-27T07:52:22.123917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('train_files:',df.shape[0])\nprint('test_files:',test_df.shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:52:25.894473Z","iopub.execute_input":"2023-11-27T07:52:25.894869Z","iopub.status.idle":"2023-11-27T07:52:25.900498Z","shell.execute_reply.started":"2023-11-27T07:52:25.894838Z","shell.execute_reply":"2023-11-27T07:52:25.899498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r /tmp/Dataset/rsna-bcd\nos.makedirs('/tmp/dataset/rsna-bcd/train_images', exist_ok = True)\nos.makedirs('/tmp/dataset/rsna-bcd/test_images', exist_ok = True)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:52:38.201164Z","iopub.execute_input":"2023-11-27T07:52:38.202133Z","iopub.status.idle":"2023-11-27T07:52:39.198858Z","shell.execute_reply.started":"2023-11-27T07:52:38.20209Z","shell.execute_reply":"2023-11-27T07:52:39.197665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\n\ndef img2roi(img):\n    # Binarize the image\n    bin_img = cv2.threshold(img, 20, 255, cv2.THRESH_BINARY)[1]\n\n    # Make contours around the binarized image, keep only the largest contour\n    contours, _ = cv2.findContours(bin_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n    contour = max(contours, key=cv2.contourArea)\n\n    # Find ROI from largest contour\n    ys = contour.squeeze()[:, 0]\n    xs = contour.squeeze()[:, 1]\n    roi =  img[np.min(xs):np.max(xs), np.min(ys):np.max(ys)]\n    \n    return roi","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:53:14.395397Z","iopub.execute_input":"2023-11-27T07:53:14.395781Z","iopub.status.idle":"2023-11-27T07:53:14.403184Z","shell.execute_reply.started":"2023-11-27T07:53:14.39575Z","shell.execute_reply":"2023-11-27T07:53:14.402232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pydicom\n# from pydicom.pixel_data_handlers.util import apply_voi_lut\n\n\nimport dicomsdl\n\n\ndef read_xray(path, fix_monochrome = True):\n    dicom = dicomsdl.open(path)\n    data = dicom.pixelData(storedvalue=False)  # storedvalue = True for int16 return otherwise float32\n    data = data - np.min(data)\n    data = data / np.max(data)\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = 1.0 - data\n    return data\n\ndef resize_and_save(file_path):\n    img = read_xray(file_path)\n    h, w = img.shape[:2]  # orig hw\n    if CFG.aspect_ratio:\n        r = CFG.resize_dim / max(h, w)  # resize image to img_size\n        interp = cv2.INTER_LINEAR\n        if r != 1:  # always resize down, only resize up if training with augmentation\n            img = cv2.resize(img, (int(w * r), int(h * r)), interpolation=interp)\n    else:\n        img = cv2.resize(img, (CFG.resize_dim, CFG.resize_dim), cv2.INTER_LINEAR)\n    \n    img = (img * 255).astype(np.uint8)\n    img = img2roi(img)\n    img = cv2.resize(img, CFG.img_size[::-1], cv2.INTER_LINEAR)\n    \n    sub_path = file_path.split(\"/\",4)[-1].split('.dcm')[0] + '.png'\n    infos = sub_path.split('/')\n    pid = infos[-2]\n    iid = infos[-1]; iid = iid.replace('.png','')\n    new_path = os.path.join(IMG_DIR, sub_path)\n    os.makedirs(new_path.rsplit('/',1)[0], exist_ok=True)\n    cv2.imwrite(new_path, img)\n    return pid,iid,w,h","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:53:16.970467Z","iopub.execute_input":"2023-11-27T07:53:16.971366Z","iopub.status.idle":"2023-11-27T07:53:16.991442Z","shell.execute_reply.started":"2023-11-27T07:53:16.971327Z","shell.execute_reply":"2023-11-27T07:53:16.990383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom joblib import Parallel, delayed\nfile_paths = test_df.dicom_path.tolist()\nimgsize = Parallel(n_jobs=2,backend='threading')(delayed(resize_and_save)(file_path)\\\n                                                  for file_path in tqdm(file_paths, leave=True, position=0))","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:53:19.871882Z","iopub.execute_input":"2023-11-27T07:53:19.872729Z","iopub.status.idle":"2023-11-27T07:53:22.569008Z","shell.execute_reply.started":"2023-11-27T07:53:19.872687Z","shell.execute_reply":"2023-11-27T07:53:22.568008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mat(shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    #rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n#     c1   = tf.math.cos(rotation)\n#     s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n#     rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n#                                    -s1,  c1,   zero, \n#                                    zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                               zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n\n    return  K.dot(shear_matrix,K.dot(zoom_matrix, shift_matrix)) #K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))                  \n\ndef transform(image, DIM=CFG.img_size):#[rot,shr,h_zoom,w_zoom,h_shift,w_shift]):\n    if DIM[0]>DIM[1]:\n        diff  = (DIM[0]-DIM[1])\n        pad   = [diff//2, diff//2 + diff%2]\n        image = tf.pad(image, [[0, 0], [pad[0], pad[1]],[0, 0]])\n        NEW_DIM = DIM[0]\n    elif DIM[0]<DIM[1]:\n        diff  = (DIM[1]-DIM[0])\n        pad   = [diff//2, diff//2 + diff%2]\n        image = tf.pad(image, [[pad[0], pad[1]], [0, 0],[0, 0]])\n        NEW_DIM = DIM[1]\n    \n    rot = CFG.rot * tf.random.normal([1], dtype='float32')\n    shr = CFG.shr * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / CFG.hzoom\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / CFG.wzoom\n    h_shift = CFG.hshift * tf.random.normal([1], dtype='float32') \n    w_shift = CFG.wshift * tf.random.normal([1], dtype='float32') \n    \n    transformation_matrix=tf.linalg.inv(get_mat(shr,h_zoom,w_zoom,h_shift,w_shift))\n    \n    flat_tensor=tfa.image.transform_ops.matrices_to_flat_transforms(transformation_matrix)\n    \n    image=tfa.image.transform(image,flat_tensor, fill_mode=CFG.fill_mode)\n    \n    rotation = math.pi * rot / 180.\n    \n    image=tfa.image.rotate(image,-rotation, fill_mode=CFG.fill_mode)\n    \n    if DIM[0]>DIM[1]:\n        image=tf.reshape(image, [NEW_DIM, NEW_DIM,3])\n        image = image[:, pad[0]:-pad[1],:]\n    elif DIM[1]>DIM[0]:\n        image=tf.reshape(image, [NEW_DIM, NEW_DIM,3])\n        image = image[pad[0]:-pad[1],:,:]\n    image = tf.reshape(image, [*DIM, 3])    \n    return image\n\ndef dropout(image,DIM=CFG.img_size, PROBABILITY = 0.6, CT = 5, SZ = 0.1):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): \n        return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*min(DIM),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3], dtype = image.dtype) \n        three = image[ya:yb,xb:DIM[1],:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n        image = tf.reshape(image,[*DIM,3])\n\n#     image = tf.reshape(image,[*DIM,3])\n    return image","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:53:28.140579Z","iopub.execute_input":"2023-11-27T07:53:28.141071Z","iopub.status.idle":"2023-11-27T07:53:28.173215Z","shell.execute_reply.started":"2023-11-27T07:53:28.14104Z","shell.execute_reply":"2023-11-27T07:53:28.172252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_decoder(with_labels=True, target_size=CFG.img_size, ext='png'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n#         img = tf.image.resize(img, target_size)\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.reshape(img, [*target_size, 3])\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), tf.cast(label, tf.float32)\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True, dim=CFG.img_size):\n    def augment(img, dim=dim):\n        img = transform(img,DIM=dim) if CFG.transform else img\n        img = tf.image.random_flip_left_right(img) if CFG.hflip else img\n        img = tf.image.random_flip_up_down(img) if CFG.vflip else img\n        if CFG.pixel_aug:\n            img = tf.image.random_hue(img, CFG.hue)\n            img = tf.image.random_saturation(img, CFG.sat[0], CFG.sat[1])\n            img = tf.image.random_contrast(img, CFG.cont[0], CFG.cont[1])\n            img = tf.image.random_brightness(img, CFG.bri)\n        img = dropout(img, DIM=dim, PROBABILITY = CFG.drop_prob, CT = CFG.drop_cnt, SZ = CFG.drop_size)\n        img = tf.clip_by_value(img, 0, 1)  if CFG.clip else img         \n        img = tf.reshape(img, [*dim, 3])\n        return img\n    \n    def augment_with_labels(img, label):    \n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, batch_size=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\", drop_remainder=False):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    ds = tf.data.Dataset.from_tensor_slices(slices)\n    ds = ds.map(decode_fn, num_parallel_calls=AUTO)\n    ds = ds.cache(cache_dir) if cache else ds\n    ds = ds.repeat() if repeat else ds\n    if shuffle: \n        ds = ds.shuffle(shuffle, seed=CFG.seed)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n    ds = ds.map(augment_fn, num_parallel_calls=AUTO) if augment else ds\n    ds = ds.batch(batch_size, drop_remainder=drop_remainder)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:53:33.541016Z","iopub.execute_input":"2023-11-27T07:53:33.541421Z","iopub.status.idle":"2023-11-27T07:53:33.562682Z","shell.execute_reply.started":"2023-11-27T07:53:33.541388Z","shell.execute_reply":"2023-11-27T07:53:33.561752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_batch(batch, size=2):\n    imgs = batch\n    plt.figure(figsize=(size*2, 10))\n    for img_idx in range(size):\n        plt.subplot(1, size, img_idx+1)\n        plt.imshow(imgs[img_idx,:, :, :])\n        plt.xticks([])\n        plt.yticks([])\n    plt.tight_layout()\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:53:38.099469Z","iopub.execute_input":"2023-11-27T07:53:38.10008Z","iopub.status.idle":"2023-11-27T07:53:38.106906Z","shell.execute_reply.started":"2023-11-27T07:53:38.100044Z","shell.execute_reply":"2023-11-27T07:53:38.105845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 0\nfold_df = test_df.copy()\npaths  = fold_df.image_path.tolist()\nlabels = None\nds = build_dataset(paths, labels, cache=False, batch_size=32,\n                   repeat=True, shuffle=True, augment=True)\nds = ds.unbatch().batch(20)\nbatch = next(iter(ds))\ndisplay_batch(batch, 5);","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:53:40.539074Z","iopub.execute_input":"2023-11-27T07:53:40.539899Z","iopub.status.idle":"2023-11-27T07:53:49.422256Z","shell.execute_reply.started":"2023-11-27T07:53:40.539863Z","shell.execute_reply":"2023-11-27T07:53:49.421225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CKPT_DIRS = [\n    ([2048, 1024], '/kaggle/input/rsnabcd-efficientnet-tftpu1vmtrain-ds'),\n]\n\nMODEL_CONFIGS = []\nfor img_size, ckpt_dir in  CKPT_DIRS:\n    paths = sorted(glob(os.path.join(ckpt_dir, '*h5')))[0:CFG.num_folds]\n    if len(paths)==0:\n        print('no model found for :',base_dir)\n    MODEL_CONFIGS.append([img_size, paths])\ndisplay(MODEL_CONFIGS)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:53:49.424159Z","iopub.execute_input":"2023-11-27T07:53:49.424797Z","iopub.status.idle":"2023-11-27T07:53:49.439622Z","shell.execute_reply.started":"2023-11-27T07:53:49.424758Z","shell.execute_reply":"2023-11-27T07:53:49.438632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_flip_model(model_path):\n    base = tf.keras.models.load_model(model_path, compile=False)\n    inp1 = base.input\n    inp2 = tf.keras.layers.Lambda(lambda x: tf.image.flip_left_right(x), name='flip')(inp1)\n    out1 = base(inp1)\n    out2 = base(inp2)\n    out = tf.keras.layers.Average(name='ensemble')([out1, out2])\n    model = tf.keras.models.Model(inp1, out)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:54:09.929095Z","iopub.execute_input":"2023-11-27T07:54:09.929965Z","iopub.status.idle":"2023-11-27T07:54:09.936751Z","shell.execute_reply.started":"2023-11-27T07:54:09.929929Z","shell.execute_reply":"2023-11-27T07:54:09.935764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('='*35)\nprint('### Predicting')\nprint('='*35)\npreds=[]\nfor idx, (img_size, model_paths) in enumerate(MODEL_CONFIGS):\n    dim = np.prod(img_size)**0.5\n    test_paths = test_df.image_path.tolist()\n    if len(test_paths)<=4:\n        CFG.batch_size = 1\n    elif dim>=1024:\n        CFG.batch_size = REPLICAS * int(4 * 2)\n    elif dim>=768:\n        CFG.batch_size = REPLICAS * int(16 * 2)\n    elif dim>=640:\n        CFG.batch_size = REPLICAS * int(28 * 2)\n    else:\n        CFG.batch_size = REPLICAS * int(32 * 2)\n    dtest = build_dataset(\n        test_paths, \n        batch_size=CFG.batch_size, repeat=True, \n        shuffle=False, augment=CFG.tta>1, cache=False,\n        decode_fn=build_decoder(with_labels=False, target_size=img_size),\n        augment_fn=build_augmenter(with_labels=False, dim=img_size)\n    )\n    print(f'Model {idx:02d}:')\n    for model_path in model_paths:\n        with strategy.scope():\n            print('Loading Model...')\n            if CFG.flip_model:\n                model = build_flip_model(model_path)\n            else:\n                model = tf.keras.models.load_model(model_path, compile=False)\n        print('Predicting...');\n        pred = model.predict(dtest, steps = CFG.tta*len(test_paths)/CFG.batch_size, verbose=1)\n        pred = pred[:CFG.tta*len(test_paths),:]\n        pred = np.mean(pred.reshape(CFG.tta, len(test_paths), -1), axis=0)\n        preds.append(pred)\n        print()\npreds = np.mean(preds, axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:54:12.635448Z","iopub.execute_input":"2023-11-27T07:54:12.636391Z","iopub.status.idle":"2023-11-27T07:54:26.44741Z","shell.execute_reply.started":"2023-11-27T07:54:12.636354Z","shell.execute_reply":"2023-11-27T07:54:26.446354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.DataFrame({'prediction_id':test_df.prediction_id,\n                        'cancer':preds.reshape(-1)})\nsub_df = pd.read_csv(f'{BASE_PATH}/sample_submission.csv')\ndel sub_df['cancer']\nsub_df = sub_df.merge(pred_df, on='prediction_id', how='left')\nsub_df = sub_df.groupby('prediction_id')['cancer'].max().reset_index()  # merge duplicate prediction_id\nsub_df['cancer'] = (sub_df.cancer>CFG.thr).astype('float32') # thresholding\nsub_df.to_csv('submission.csv',index=False)\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T07:54:26.493185Z","iopub.execute_input":"2023-11-27T07:54:26.493574Z","iopub.status.idle":"2023-11-27T07:54:26.516603Z","shell.execute_reply.started":"2023-11-27T07:54:26.493513Z","shell.execute_reply":"2023-11-27T07:54:26.515571Z"},"trusted":true},"execution_count":null,"outputs":[]}]}