{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":52254,"databundleVersionId":8756537,"sourceType":"competition"},{"sourceId":6461898,"sourceType":"datasetVersion","datasetId":3731763},{"sourceId":6462974,"sourceType":"datasetVersion","datasetId":3732424}],"dockerImageVersionId":30527,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q keras-cv-attention-models\n!pip install -qU scikit-learn\n!pip install -q seaborn","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-15T05:00:57.391494Z","iopub.execute_input":"2024-09-15T05:00:57.391792Z","iopub.status.idle":"2024-09-15T05:01:37.004245Z","shell.execute_reply.started":"2024-09-15T05:00:57.391765Z","shell.execute_reply":"2024-09-15T05:01:37.002841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -qU wandb==0.15.8","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:01:37.006715Z","iopub.execute_input":"2024-09-15T05:01:37.007438Z","iopub.status.idle":"2024-09-15T05:01:51.343803Z","shell.execute_reply.started":"2024-09-15T05:01:37.007396Z","shell.execute_reply":"2024-09-15T05:01:51.342552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'  # to avoid too many logging messages\nimport pandas as pd, numpy as np, random, shutil\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport sklearn\n\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nimport tensorflow_probability as tfp\nimport wandb\nimport yaml\n\nfrom IPython import display as ipd\nfrom glob import glob\nfrom tqdm import tqdm\nfrom sklearn.model_selection import KFold, StratifiedKFold, GroupKFold, StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom tensorflow import keras","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:01:51.345391Z","iopub.execute_input":"2024-09-15T05:01:51.345696Z","iopub.status.idle":"2024-09-15T05:02:01.461485Z","shell.execute_reply.started":"2024-09-15T05:01:51.345667Z","shell.execute_reply":"2024-09-15T05:02:01.460717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('np:', np.__version__)\nprint('pd:', pd.__version__)\nprint('sklearn:', sklearn.__version__)\nprint('tf:',tf.__version__)\nprint('tfp:', tfp.__version__)\nprint('tfa:', tfa.__version__)\nprint('w&b:', wandb.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:01.464139Z","iopub.execute_input":"2024-09-15T05:02:01.464865Z","iopub.status.idle":"2024-09-15T05:02:01.470904Z","shell.execute_reply.started":"2024-09-15T05:02:01.464828Z","shell.execute_reply":"2024-09-15T05:02:01.469881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Wandb 사용**","metadata":{}},{"cell_type":"code","source":"import wandb\n\ntry:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    api_key = user_secrets.get_secret(\"WANDB\")\n\n    wandb.login(key=api_key)\n    anonymous = None\nexcept:\n    anonymous = \"must\"\n    print('https://wandb.ai/authorize')","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:01.472211Z","iopub.execute_input":"2024-09-15T05:02:01.472562Z","iopub.status.idle":"2024-09-15T05:02:01.823303Z","shell.execute_reply.started":"2024-09-15T05:02:01.47253Z","shell.execute_reply":"2024-09-15T05:02:01.822341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **구성(Config)**","metadata":{}},{"cell_type":"code","source":"class CFG:\n    wandb         = True\n    competition   = 'rsna-atd' \n    _wandb_kernel = 'awsaf49'\n    debug         = False\n    comment       = 'Resnest-512x512-3window'\n    exp_name      = 'resnest50-png' # name of the experiment, folds will be grouped using 'exp_name'\n    \n    # use verbose=0 for silent, vebose=1 for interactive,\n    verbose      = 0\n    display_plot = True\n\n    # device\n    device = \"TPU-VM\" #or \"GPU\"\n\n    model_name = 'ResNest50'\n\n    # seed for data-split, layer init, augs\n    seed = 19\n\n    # number of folds for data-split\n    folds = 4\n    \n    # which folds to train\n    selected_folds = [0, 1, 2]\n\n    # size of the image\n    img_size = [512, 512]\n#     eq_dim = np.prod(img_size)**0.5\n\n    # batch_size and epochs\n    batch_size = 16\n    epochs = 50\n\n    # loss\n    loss      = 'BCE & CCE'  # BCE, Focal\n    \n    # optimizer\n    optimizer = 'Adam'\n\n    # augmentation\n    augment   = True\n\n    # scale-shift-rotate-shear\n    transform = 0.90  # transform prob\n    fill_mode = 'constant'\n    rot    = 2.0\n    shr    = 2.0\n    hzoom  = 50.0\n    wzoom  = 50.0\n    hshift = 10.0\n    wshift = 10.0\n\n    # flip\n    hflip = True\n    vflip = True\n\n    # clip\n    clip = False\n\n    # lr-scheduler\n    scheduler   = 'cosine' # cosine\n\n    # dropout\n    drop_prob   = 0.6\n    drop_cnt    = 5\n    drop_size   = 0.05\n    \n    # cut-mix-up\n    mixup_prob = 0.0\n    mixup_alpha = 0.5\n    \n    cutmix_prob = 0.0\n    cutmix_alpha = 2.5\n\n    # pixel-augment\n    pixel_aug = 0.90  # prob of pixel_aug\n    sat  = [0.7, 1.3]\n    cont = [0.8, 1.2]\n    bri  = 0.15\n    hue  = 0.05\n\n    # test-time augs\n    tta = 1\n    \n    # target column\n    target_col  = [ \"bowel_injury\", \"extravasation_injury\", \"kidney_healthy\", \"kidney_low\",\n                   \"kidney_high\", \"liver_healthy\", \"liver_low\", \"liver_high\",\n                   \"spleen_healthy\", \"spleen_low\", \"spleen_high\"] # not using \"bowel_healthy\" & \"extravasation_healthy\"","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:01.824453Z","iopub.execute_input":"2024-09-15T05:02:01.824757Z","iopub.status.idle":"2024-09-15T05:02:01.835976Z","shell.execute_reply.started":"2024-09-15T05:02:01.824732Z","shell.execute_reply":"2024-09-15T05:02:01.835101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n#     os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)\n    tf.random.set_seed(SEED)\n    print('seeding done!!!')\nseeding(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:01.837269Z","iopub.execute_input":"2024-09-15T05:02:01.837565Z","iopub.status.idle":"2024-09-15T05:02:01.850781Z","shell.execute_reply.started":"2024-09-15T05:02:01.837532Z","shell.execute_reply":"2024-09-15T05:02:01.849888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **GPU, TPU connect**","metadata":{}},{"cell_type":"code","source":"if \"TPU\" in CFG.device:\n    tpu = 'local' if CFG.device=='TPU-VM' else None\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=tpu)\n        strategy = tf.distribute.TPUStrategy(tpu)\n    except:\n        CFG.device = \"GPU\"\n        \nif CFG.device == \"GPU\"  or CFG.device==\"CPU\":\n    ngpu = len(tf.config.experimental.list_physical_devices('GPU'))\n    if ngpu>1:\n        print(\"Using multi GPU\")\n        strategy = tf.distribute.MirroredStrategy()\n    elif ngpu==1:\n        print(\"Using single GPU\")\n        strategy = tf.distribute.get_strategy()\n    else:\n        print(\"Using CPU\")\n        strategy = tf.distribute.get_strategy()\n        CFG.device = \"CPU\"\n\nif CFG.device == \"GPU\":\n    print(\"Num GPUs Available: \", ngpu)\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:01.851825Z","iopub.execute_input":"2024-09-15T05:02:01.852086Z","iopub.status.idle":"2024-09-15T05:02:04.069306Z","shell.execute_reply.started":"2024-09-15T05:02:01.852044Z","shell.execute_reply":"2024-09-15T05:02:04.068381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **PNG 이미지 Meta Data**","metadata":{}},{"cell_type":"code","source":"# path\nBASE_PATH = f'/kaggle/input/rsna-atd-512x512-png-3window'","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:04.070394Z","iopub.execute_input":"2024-09-15T05:02:04.070684Z","iopub.status.idle":"2024-09-15T05:02:04.074734Z","shell.execute_reply.started":"2024-09-15T05:02:04.070659Z","shell.execute_reply":"2024-09-15T05:02:04.073849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##다른 사람이 PNG로 바꿔놓은 이미지 일단 사용함\n# train\ndf = pd.read_csv(f'/kaggle/input/train-new/train.csv')\ndf['image_path'] = f'/kaggle/input/rsna-atd-512x512-png-3window'\\\n                    + '/' + df.patient_id.astype(str)\\\n                    + '/' + df.series_id.astype(str)\\\n                    + '/' + df.instance_number.astype(str) +'.png' #dcm을 png로 바꿈\ndf = df.drop_duplicates()\nprint('Train:')\ndisplay(df.head(2))\n\n# test\n# test_df = pd.read_csv(f'{BASE_PATH}/test.csv')\n# test_df['image_path'] = f'{BASE_PATH}/test_images'\\\n#                     + '/' + test_df.patient_id.astype(str)\\\n#                     + '/' + test_df.series_id.astype(str)\\\n#                     + '/' + test_df.instance_number.astype(str) +'.png'\n# test_df = test_df.drop_duplicates()\n# print('\\nTest:')\n# display(test_df.head(2))","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:04.0785Z","iopub.execute_input":"2024-09-15T05:02:04.078843Z","iopub.status.idle":"2024-09-15T05:02:04.333383Z","shell.execute_reply.started":"2024-09-15T05:02:04.078819Z","shell.execute_reply":"2024-09-15T05:02:04.332452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 파일 유무 확인\ntf.io.gfile.exists(df.image_path.iloc[0])\n# tf.io.gfile.exists(test_df.image_path.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:04.334731Z","iopub.execute_input":"2024-09-15T05:02:04.335578Z","iopub.status.idle":"2024-09-15T05:02:04.350284Z","shell.execute_reply.started":"2024-09-15T05:02:04.335539Z","shell.execute_reply":"2024-09-15T05:02:04.349411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 전체 subject 확인\nprint('train_files:',df.shape[0])\n# print('test_files:',test_df.shape[0])","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:04.351214Z","iopub.execute_input":"2024-09-15T05:02:04.351456Z","iopub.status.idle":"2024-09-15T05:02:04.356359Z","shell.execute_reply.started":"2024-09-15T05:02:04.351436Z","shell.execute_reply":"2024-09-15T05:02:04.355351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Data Split(비율 유지)**","metadata":{}},{"cell_type":"code","source":"# Label 추가\ndf['stratify'] = ''\nfor col in CFG.target_col:\n    df['stratify'] += df[col].astype(str)\n\ndf = df.reset_index(drop=True) # 제거된 거 있으니까 index number 다시\nskf = StratifiedGroupKFold(n_splits=CFG.folds, shuffle=True, random_state=CFG.seed) # fold 설정\n# df: 전체이미지, df['stratify']: Lable, df['patient_id']: 그룹(subject)\nfor fold, (train_idx, val_idx) in enumerate(skf.split(df, df['stratify'], df[\"patient_id\"])):\n    df.loc[val_idx, 'fold'] = fold #valid fold number\ndisplay(df.groupby(['fold', 'patient_id']).size())","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:04.357552Z","iopub.execute_input":"2024-09-15T05:02:04.357822Z","iopub.status.idle":"2024-09-15T05:02:04.713377Z","shell.execute_reply.started":"2024-09-15T05:02:04.357799Z","shell.execute_reply":"2024-09-15T05:02:04.712423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Augmentation**","metadata":{}},{"cell_type":"code","source":"def get_mat(shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    #rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n#     c1   = tf.math.cos(rotation)\n#     s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n#     rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n#                                    -s1,  c1,   zero, \n#                                    zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                               zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n\n    return  K.dot(shear_matrix,K.dot(zoom_matrix, shift_matrix)) #K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))                  \n\ndef transform(image, DIM=CFG.img_size):#[rot,shr,h_zoom,w_zoom,h_shift,w_shift]):\n    if DIM[0]>DIM[1]:\n        diff  = (DIM[0]-DIM[1])\n        pad   = [diff//2, diff//2 + diff%2]\n        image = tf.pad(image, [[0, 0], [pad[0], pad[1]],[0, 0]])\n        NEW_DIM = DIM[0]\n    elif DIM[0]<DIM[1]:\n        diff  = (DIM[1]-DIM[0])\n        pad   = [diff//2, diff//2 + diff%2]\n        image = tf.pad(image, [[pad[0], pad[1]], [0, 0],[0, 0]])\n        NEW_DIM = DIM[1]\n    \n    rot = CFG.rot * tf.random.normal([1], dtype='float32')\n    shr = CFG.shr * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / CFG.hzoom\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / CFG.wzoom\n    h_shift = CFG.hshift * tf.random.normal([1], dtype='float32') \n    w_shift = CFG.wshift * tf.random.normal([1], dtype='float32') \n    \n    transformation_matrix=tf.linalg.inv(get_mat(shr,h_zoom,w_zoom,h_shift,w_shift))\n    \n    flat_tensor=tfa.image.transform_ops.matrices_to_flat_transforms(transformation_matrix)\n    \n    image=tfa.image.transform(image,flat_tensor, fill_mode=CFG.fill_mode)\n    \n    rotation = math.pi * rot / 180.\n    \n    image=tfa.image.rotate(image,-rotation, fill_mode=CFG.fill_mode)\n    \n    if DIM[0]>DIM[1]:\n        image=tf.reshape(image, [NEW_DIM, NEW_DIM,3])\n        image = image[:, pad[0]:-pad[1],:]\n    elif DIM[1]>DIM[0]:\n        image=tf.reshape(image, [NEW_DIM, NEW_DIM,3])\n        image = image[pad[0]:-pad[1],:,:]\n    image = tf.reshape(image, [*DIM, 3])    \n    return image\n\ndef dropout(image,DIM=CFG.img_size, PROBABILITY = 0.6, CT = 5, SZ = 0.1):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): \n        return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*min(DIM),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3], dtype = image.dtype) \n        three = image[ya:yb,xb:DIM[1],:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n        image = tf.reshape(image,[*DIM,3])\n\n#     image = tf.reshape(image,[*DIM,3])\n    return image","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:04.714805Z","iopub.execute_input":"2024-09-15T05:02:04.715126Z","iopub.status.idle":"2024-09-15T05:02:04.739662Z","shell.execute_reply.started":"2024-09-15T05:02:04.7151Z","shell.execute_reply":"2024-09-15T05:02:04.738846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **이미지 변환**","metadata":{}},{"cell_type":"code","source":"def build_decoder(with_labels=True, target_size=CFG.img_size, ext='png'):\n    def decode_image(path): # 이미지 읽고 RGB로 디코딩\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3, dtype=tf.uint8)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n        img = tf.image.resize(img, target_size, method='bilinear')\n        img = tf.cast(img, tf.float32) / 255.0 #float32로 타입 변경, 0-1로 normalize\n        img = tf.reshape(img, [*target_size, 3]) #한번더 정렬\n\n        return img\n     \n    def decode_label(label): #레이블도 float으로\n        label = tf.cast(label, tf.float32)\n        return (label[0:1], label[1:2], label[2:5], label[5:8], label[8:11])\n    \n    def decode_with_labels(path, label): \n        return decode_image(path), decode_label(label)\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True, dim=CFG.img_size): #변환, augment\n    def augment(img, dim=dim):\n        if random.random() < CFG.transform: #(0.9)\n            img = transform(img,DIM=dim) #[rot,shr,h_zoom,w_zoom,h_shift,w_shift])\n        img = tf.image.random_flip_left_right(img) if CFG.hflip else img\n        img = tf.image.random_flip_up_down(img) if CFG.vflip else img\n        if random.random() < CFG.pixel_aug:\n            img = tf.image.random_hue(img, CFG.hue)\n            img = tf.image.random_saturation(img, CFG.sat[0], CFG.sat[1])\n            img = tf.image.random_contrast(img, CFG.cont[0], CFG.cont[1])\n            img = tf.image.random_brightness(img, CFG.bri)\n        img = tf.clip_by_value(img, 0, 1)  if CFG.clip else img         \n        img = tf.reshape(img, [*dim, 3])\n        return img\n    \n    def augment_with_labels(img, label):    \n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, batch_size=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\", drop_remainder=False):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True) #디렉토리 생성\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    ds = tf.data.Dataset.from_tensor_slices(slices)\n    ds = ds.map(decode_fn, num_parallel_calls=AUTO)\n    ds = ds.cache(cache_dir) if cache else ds\n    ds = ds.repeat() if repeat else ds\n    if shuffle: \n        ds = ds.shuffle(shuffle, seed=CFG.seed)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n    ds = ds.map(augment_fn, num_parallel_calls=AUTO) if augment else ds\n    if augment and labels is not None:\n        ds = ds.map(lambda img, label: (dropout(img, \n                                               DIM=CFG.img_size, \n                                               PROBABILITY=CFG.drop_prob, \n                                               CT=CFG.drop_cnt,\n                                               SZ=CFG.drop_size), label),num_parallel_calls=AUTO)\n    ds = ds.batch(batch_size, drop_remainder=drop_remainder)\n    if augment and labels is not None:\n        if CFG.cutmix_prob:\n            ds = ds.map(get_cutmix(alpha=CFG.cutmix_alpha,prob=CFG.cutmix_prob),num_parallel_calls=AUTO)\n        if CFG.mixup_prob:\n            ds = ds.map(get_mixup(alpha=CFG.mixup_alpha,prob=CFG.mixup_prob),num_parallel_calls=AUTO)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:04.741003Z","iopub.execute_input":"2024-09-15T05:02:04.741387Z","iopub.status.idle":"2024-09-15T05:02:04.763311Z","shell.execute_reply.started":"2024-09-15T05:02:04.741361Z","shell.execute_reply":"2024-09-15T05:02:04.762543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Image 변환 확인**","metadata":{}},{"cell_type":"code","source":"def display_batch(batch, size=2):\n    if isinstance(batch, tuple):\n        imgs, tars = batch\n    else:\n        imgs = batch\n        tars = None\n    tars = tf.concat(tars,axis=-1).numpy()\n    plt.figure(figsize=(size*5, 10))\n    for img_idx in range(size):\n        plt.subplot(1, size, img_idx+1)\n        if tars is not None:\n            plt.title(f'{tars[img_idx].round(2)}', fontsize=12)\n        img = imgs[img_idx,]\n        plt.imshow(img)\n        plt.xticks([]); plt.yticks([])\n    plt.tight_layout()\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:04.764577Z","iopub.execute_input":"2024-09-15T05:02:04.764979Z","iopub.status.idle":"2024-09-15T05:02:04.776619Z","shell.execute_reply.started":"2024-09-15T05:02:04.764908Z","shell.execute_reply":"2024-09-15T05:02:04.775803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 0\nfold_df = df[df.fold==fold].sample(frac=1.0)\npaths  = fold_df.image_path.tolist()\nlabels = fold_df[CFG.target_col].values\nds = build_dataset(paths, labels, cache=False, batch_size=32,\n                   repeat=True, shuffle=True, augment=False)\nds = ds.unbatch().batch(20)\nbatch = next(iter(ds))","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:04.777738Z","iopub.execute_input":"2024-09-15T05:02:04.778044Z","iopub.status.idle":"2024-09-15T05:02:05.906267Z","shell.execute_reply.started":"2024-09-15T05:02:04.778003Z","shell.execute_reply":"2024-09-15T05:02:05.905415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_batch(batch, 3);","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:05.907441Z","iopub.execute_input":"2024-09-15T05:02:05.907738Z","iopub.status.idle":"2024-09-15T05:02:06.652038Z","shell.execute_reply.started":"2024-09-15T05:02:05.907714Z","shell.execute_reply":"2024-09-15T05:02:06.651126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('약간씩 틀어지는 모양')\ntimgs = tf.map_fn(lambda img: transform(img,DIM=CFG.img_size), batch[0])\nttars = batch[1]\ndisplay_batch((timgs, ttars), 3);","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:06.653285Z","iopub.execute_input":"2024-09-15T05:02:06.653583Z","iopub.status.idle":"2024-09-15T05:02:11.215153Z","shell.execute_reply.started":"2024-09-15T05:02:06.653558Z","shell.execute_reply":"2024-09-15T05:02:11.214201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Model**","metadata":{}},{"cell_type":"code","source":"from keras_cv_attention_models import resnest\n\ndef build_model(model_name=CFG.model_name,\n                loss_name=CFG.loss,\n                dim=CFG.img_size,\n                compile_model=True,\n                include_top=False):         \n    \n    # Define backbone\n    base = getattr(resnest, model_name)(input_shape=(*dim,3),\n                                    pretrained='imagenet',\n                                    num_classes=0) # get base model (efficientnet), use imgnet weights\n\n    inp = base.inputs\n    x = base.output\n    x = tf.keras.layers.GlobalAveragePooling2D(name='GAP')(x) # use GAP to get pooling result form conv outputs\n\n    \n    x_bowel = tf.keras.layers.Dense(1024, activation='silu')(x)\n    x_extra = tf.keras.layers.Dense(1024, activation='silu')(x)\n    x_liver = tf.keras.layers.Dense(1024, activation='silu')(x)\n    x_kidney = tf.keras.layers.Dense(1024, activation='silu')(x)\n    x_spleen = tf.keras.layers.Dense(1024, activation='silu')(x)\n    \n    x_bowel = tf.keras.layers.Dense(512, activation='silu')(x_bowel)\n    x_extra = tf.keras.layers.Dense(512, activation='silu')(x_extra)\n    x_liver = tf.keras.layers.Dense(512, activation='silu')(x_liver)\n    x_kidney = tf.keras.layers.Dense(512, activation='silu')(x_kidney)\n    x_spleen = tf.keras.layers.Dense(512, activation='silu')(x_spleen)\n\n    x_bowel = tf.keras.layers.Dense(128, activation='silu')(x_bowel)\n    x_extra = tf.keras.layers.Dense(128, activation='silu')(x_extra)\n    x_liver = tf.keras.layers.Dense(128, activation='silu')(x_liver)\n    x_kidney = tf.keras.layers.Dense(128, activation='silu')(x_kidney)\n    x_spleen = tf.keras.layers.Dense(128, activation='silu')(x_spleen)\n\n    x_bowel = tf.keras.layers.Dense(64, activation='silu')(x_bowel)\n    x_extra = tf.keras.layers.Dense(64, activation='silu')(x_extra)\n    x_liver = tf.keras.layers.Dense(64, activation='silu')(x_liver)\n    x_kidney = tf.keras.layers.Dense(64, activation='silu')(x_kidney)\n    x_spleen = tf.keras.layers.Dense(64, activation='silu')(x_spleen)\n    \n    x_bowel = tf.keras.layers.Dense(32, activation='silu')(x_bowel)\n    x_extra = tf.keras.layers.Dense(32, activation='silu')(x_extra)\n    x_liver = tf.keras.layers.Dense(32, activation='silu')(x_liver)\n    x_kidney = tf.keras.layers.Dense(32, activation='silu')(x_kidney)\n    x_spleen = tf.keras.layers.Dense(32, activation='silu')(x_spleen)\n    \n    x_bowel = tf.keras.layers.Dense(16, activation='silu')(x_bowel)\n    x_extra = tf.keras.layers.Dense(16, activation='silu')(x_extra)\n    x_liver = tf.keras.layers.Dense(16, activation='silu')(x_liver)\n    x_kidney = tf.keras.layers.Dense(16, activation='silu')(x_kidney)\n    x_spleen = tf.keras.layers.Dense(16, activation='silu')(x_spleen)\n    # Define heads\n    out_bowel = tf.keras.layers.Dense(1, name='bowel', activation='sigmoid')(x_bowel) # use sigmoid to convert predictions to [0-1]\n    out_extra = tf.keras.layers.Dense(1, name='extra', activation='sigmoid')(x_extra) # use sigmoid to convert predictions to [0-1]\n    out_liver = tf.keras.layers.Dense(3, name='liver', activation='softmax')(x_liver) # use softmax for the liver head\n    out_kidney = tf.keras.layers.Dense(3, name='kidney', activation='softmax')(x_kidney) # use softmax for the kidney head\n    out_spleen = tf.keras.layers.Dense(3, name='spleen', activation='softmax')(x_spleen) # u\n    \n    # Combine outputs\n#     out = tf.keras.layers.Concatenate()([out_bowel, out_extra, \n#                                          out_liver, out_kidney, out_spleen])\n    out = [out_bowel, out_extra, out_kidney, out_liver, out_spleen]\n\n    # Create model\n    model = tf.keras.Model(inputs=inp, outputs=out)\n\n    \n    if compile_model:\n        # optimizer\n        opt = tf.keras.optimizers.Adam(learning_rate=0.000001)\n        # loss\n        loss = {\n            'bowel':tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05),\n            'extra':tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05),\n            'liver':tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),\n            'kidney':tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),\n            'spleen':tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),\n        }\n        # metric\n        metrics = {\n            'bowel':['accuracy'],\n            'extra':['accuracy'],\n            'liver':['accuracy'],\n            'kidney':['accuracy'],\n            'spleen':['accuracy'],\n        }\n        # compile\n        model.compile(optimizer=opt,\n                      loss=loss,\n                      metrics=metrics)\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:11.216509Z","iopub.execute_input":"2024-09-15T05:02:11.216808Z","iopub.status.idle":"2024-09-15T05:02:11.380073Z","shell.execute_reply.started":"2024-09-15T05:02:11.216782Z","shell.execute_reply":"2024-09-15T05:02:11.379098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = build_model(CFG.model_name, dim=CFG.img_size, compile_model=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:11.381272Z","iopub.execute_input":"2024-09-15T05:02:11.381561Z","iopub.status.idle":"2024-09-15T05:02:17.874931Z","shell.execute_reply.started":"2024-09-15T05:02:11.381537Z","shell.execute_reply":"2024-09-15T05:02:17.873959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(batch_size=8, plot=False):\n    lr_start   = 0.000005\n    lr_max     = 0.00000050 * REPLICAS * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 4\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        elif CFG.scheduler=='exp':\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        elif CFG.scheduler=='cosine':\n            decay_total_epochs = CFG.epochs - lr_ramp_ep - lr_sus_ep + 3\n            decay_epoch_index = epoch - lr_ramp_ep - lr_sus_ep\n            phase = math.pi * decay_epoch_index / decay_total_epochs\n            cosine_decay = 0.4 * (1 + math.cos(phase))\n            lr = (lr_max - lr_min) * cosine_decay + lr_min\n        return lr\n    if plot:\n        plt.figure(figsize=(10,5))\n        plt.plot(np.arange(CFG.epochs), [lrfn(epoch) for epoch in np.arange(CFG.epochs)], marker='o')\n        plt.xlabel('epoch'); plt.ylabel('learnig rate')\n        plt.title('Learning Rate Scheduler')\n        plt.show()\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\n_=get_lr_callback(CFG.batch_size, plot=True )","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:17.876288Z","iopub.execute_input":"2024-09-15T05:02:17.876585Z","iopub.status.idle":"2024-09-15T05:02:18.168258Z","shell.execute_reply.started":"2024-09-15T05:02:17.876559Z","shell.execute_reply":"2024-09-15T05:02:18.167247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create directory to save gradcam imgs\n!mkdir -p gradcam\n\n# intialize wandb run\ndef wandb_init(fold):\n    config = {k:v for k,v in dict(vars(CFG)).items() if '__' not in k}\n    config.update({\"fold\":int(fold)})\n    yaml.dump(config, open(f'/kaggle/working/config fold-{fold}.yaml', 'w'),)\n    config = yaml.load(open(f'/kaggle/working/config fold-{fold}.yaml', 'r'), Loader=yaml.FullLoader)\n    run    = wandb.init(project=\"rsna-atd-public\",\n               name=f\"fold-{fold}|dim-{CFG.img_size[0]}x{CFG.img_size[1]}|model-{CFG.model_name}\",\n               config=config,\n               anonymous=anonymous,\n               group=CFG.exp_name\n                    )\n    return run\n\ndef log_wandb(fold):\n    \"log best result for error analysis\"\n    # log values to wandb\n    wandb.log({\n               'best_acc': best_acc,\n               'best_loss': best_loss,\n               'best_epoch': best_epoch,\n               'best_acc_bowel': best_acc_bowel,\n               'best_acc_extra': best_acc_extra,\n               'best_acc_liver': best_acc_liver,\n               'best_acc_kidney': best_acc_kidney,\n               'best_acc_spleen': best_acc_spleen,\n              })","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:18.169511Z","iopub.execute_input":"2024-09-15T05:02:18.169807Z","iopub.status.idle":"2024-09-15T05:02:19.212975Z","shell.execute_reply.started":"2024-09-15T05:02:18.169782Z","shell.execute_reply":"2024-09-15T05:02:19.211812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get wandb callbacks\ndef get_wb_callbacks(fold):\n    wb_ckpt = wandb.keras.WandbModelCheckpoint(filepath='fold-%i.h5'%fold, \n                                               monitor='val_loss',\n                                               verbose=CFG.verbose,\n                                               save_best_only=True,\n                                               save_weights_only=False,\n                                               mode='min',)\n    wb_metr = wandb.keras.WandbMetricsLogger()\n    return [wb_ckpt, wb_metr]","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:19.214724Z","iopub.execute_input":"2024-09-15T05:02:19.215741Z","iopub.status.idle":"2024-09-15T05:02:19.221732Z","shell.execute_reply.started":"2024-09-15T05:02:19.215696Z","shell.execute_reply":"2024-09-15T05:02:19.220722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []\n\nfor fold in np.arange(CFG.folds):\n    \n    # ignore not selected folds\n    if fold not in CFG.selected_folds:\n        continue\n        \n    # init wandb\n    if CFG.wandb:\n        run = wandb_init(fold)\n        wb_callbacks = get_wb_callbacks(fold)\n            \n    # train and valid dataframe\n    train_df = df.query(\"fold!=@fold\")\n    valid_df = df.query(\"fold==@fold\")\n    \n    # get image_paths and labels\n    train_paths = train_df.image_path.values; train_labels = train_df[CFG.target_col].values.astype(np.float32)\n    valid_paths = valid_df.image_path.values; valid_labels = valid_df[CFG.target_col].values.astype(np.float32)\n#     test_paths  = test_df.image_path.values\n    \n    # shuffle train data\n    index = np.arange(len(train_df))\n    np.random.shuffle(index)\n    train_paths  = train_paths[index]\n    train_labels = train_labels[index]\n    \n    # min samples in debug mode\n    min_samples = CFG.batch_size*REPLICAS*2\n    \n    # for debug model run on small portion\n    if CFG.debug:\n        train_paths = train_paths[:min_samples]; train_labels = train_labels[:min_samples]\n        valid_paths = valid_paths[:min_samples]; valid_labels = valid_labels[:min_samples]\n    \n    # show message\n    print('#'*40); print('#### FOLD: ',fold)\n    print('#### IMAGE_SIZE: (%i, %i) | MODEL_NAME: %s | BATCH_SIZE: %i'%\n          (CFG.img_size[0],CFG.img_size[1],CFG.model_name,CFG.batch_size*REPLICAS))\n    \n    # data stat\n    num_train = len(train_paths)\n    num_valid = len(valid_paths)\n    if CFG.wandb:\n        wandb.log({'num_train':num_train,\n                   'num_valid':num_valid})\n    print('#### NUM_TRAIN: {:,} | NUM_VALID: {:,}'.format(num_train, num_valid))\n    \n    # build model\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(CFG.model_name, dim=CFG.img_size, compile_model=True)\n\n    # build dataset\n    cache = 1 if 'TPU' in CFG.device else 0\n    train_ds = build_dataset(train_paths, train_labels, cache=cache, batch_size=CFG.batch_size*REPLICAS,\n                   repeat=True, shuffle=True, augment=CFG.augment)\n    val_ds = build_dataset(valid_paths, valid_labels, cache=cache, batch_size=CFG.batch_size*REPLICAS,\n                   repeat=False, shuffle=False, augment=False)\n    print('#'*40)   \n    \n    # callbacks\n    callbacks = []\n    ## save best model after each fold\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=CFG.verbose, save_best_only=True,\n        save_weights_only=False, mode='min', save_freq='epoch')\n    callbacks +=[sv]\n    ## lr-scheduler\n    callbacks += [get_lr_callback(CFG.batch_size)]\n    ## wandb callbacks\n    if CFG.wandb:\n        callbacks += wb_callbacks\n        \n    earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',patience=10,verbose=0)\n        \n    # train\n    print('Training...')\n    history = model.fit(\n        train_ds, \n        epochs=CFG.epochs if not CFG.debug else 2, \n        callbacks = [callbacks,earlyStopping],  \n        steps_per_epoch=len(train_paths)/CFG.batch_size//REPLICAS,\n        validation_data=val_ds, \n        verbose=1\n    )\n    \n    # store best results\n    best_epoch = np.argmin(history.history['val_loss'])\n    best_loss = history.history['val_loss'][best_epoch]\n    best_acc_bowel = history.history['val_bowel_accuracy'][best_epoch]\n    best_acc_extra = history.history['val_extra_accuracy'][best_epoch]\n    best_acc_liver = history.history['val_liver_accuracy'][best_epoch]\n    best_acc_kidney = history.history['val_kidney_accuracy'][best_epoch]\n    best_acc_spleen = history.history['val_spleen_accuracy'][best_epoch]\n\n    # Find mean accuracy\n    best_acc = np.mean([best_acc_bowel, best_acc_extra, \n                        best_acc_liver, best_acc_kidney, best_acc_spleen])\n\n    print(f'\\n{\"=\"*17} FOLD {fold} RESULTS {\"=\"*17}')\n    print(f'>>>> BEST Loss  : {best_loss:.3f}\\n>>>> BEST Acc   : {best_acc:.3f}\\n>>>> BEST Epoch : {best_epoch}\\n')\n    print('ORGAN Acc:')\n    print(f'  >>>> {\"Bowel\".ljust(15)} : {best_acc_bowel:.3f}')\n    print(f'  >>>> {\"Extravasation\".ljust(15)} : {best_acc_extra:.3f}')\n    print(f'  >>>> {\"Liver\".ljust(15)} : {best_acc_liver:.3f}')\n    print(f'  >>>> {\"Kidney\".ljust(15)} : {best_acc_kidney:.3f}')\n    print(f'  >>>> {\"Spleen\".ljust(15)} : {best_acc_spleen:.3f}')\n\n    print(f'{\"=\"*50}\\n')\n\n    scores.append([best_loss, best_acc, \n                   best_acc_bowel, best_acc_extra, \n                   best_acc_liver, best_acc_kidney, best_acc_spleen])\n    \n    # log best result on wandb & plot\n    if CFG.wandb:\n        log_wandb(fold) # log\n        wandb.run.finish() # finish the run\n        display(ipd.IFrame(run.url, width=1080, height=720)) # show wandb dashboard","metadata":{"execution":{"iopub.status.busy":"2024-09-15T05:02:19.223384Z","iopub.execute_input":"2024-09-15T05:02:19.223694Z","iopub.status.idle":"2024-09-15T13:33:04.848548Z","shell.execute_reply.started":"2024-09-15T05:02:19.22367Z","shell.execute_reply":"2024-09-15T13:33:04.847647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# overall oof pF1\noof_loss, oof_acc, oof_acc_bowel, oof_acc_extra, oof_acc_liver, oof_acc_kidney, oof_acc_spleen = np.array(scores).mean(axis=0)\n\nprint(f'\\n{\"=\"*15} OVERALL OOF RESULTS {\"=\"*15}')\nprint(f'>>>> OOF BEST Loss : {oof_loss:.3f}\\n>>>> OOF BEST Acc  : {oof_acc:.3f}\\n')\nprint('ORGAN OOF Acc:')\nprint(f'  >>>> {\"Bowel\".ljust(15)} : {oof_acc_bowel:.3f}')\nprint(f'  >>>> {\"Extravasation\".ljust(15)} : {oof_acc_extra:.3f}')\nprint(f'  >>>> {\"Liver\".ljust(15)} : {oof_acc_liver:.3f}')\nprint(f'  >>>> {\"Kidney\".ljust(15)} : {oof_acc_kidney:.3f}')\nprint(f'  >>>> {\"Spleen\".ljust(15)} : {oof_acc_spleen:.3f}')\nprint(f'{\"=\"*50}\\n')","metadata":{"execution":{"iopub.status.busy":"2024-09-15T13:33:04.85493Z","iopub.execute_input":"2024-09-15T13:33:04.855338Z","iopub.status.idle":"2024-09-15T13:33:04.862626Z","shell.execute_reply.started":"2024-09-15T13:33:04.85531Z","shell.execute_reply":"2024-09-15T13:33:04.861789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r /kaggle/working/wandb","metadata":{"execution":{"iopub.status.busy":"2024-09-15T13:33:04.863754Z","iopub.execute_input":"2024-09-15T13:33:04.864582Z","iopub.status.idle":"2024-09-15T13:33:07.674219Z","shell.execute_reply.started":"2024-09-15T13:33:04.86455Z","shell.execute_reply":"2024-09-15T13:33:07.672897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}