{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"ENV = 'kaggle'\n\nclass CFG:\n    DS_TYPE_SINGLE = 'SINGLE'\n    DS_TYPE_MULTIPLE = 'MULTIPLE'\n    DS_TYPE_SINGLE_ASH = 'SINGLE_ASH'\n    DS_TYPE_MULTIPLE_ASHS = 'MULTIPLE_ASHS'\n\n    DS_TYPE = DS_TYPE_SINGLE_ASH\n    FLOAT_16 = True\n    BATCH_SIZE = 8\n    ORI_SIZE = 256\n    TRAIN_SIZE = 512\n    VALID_SIZE = 768\n    AUG_ROT = 0\n    AUG_JPEG = None\n    AUG_COLOR = None\n    AUG_NOISE = None\n#     AUG_FLIP_LR = None\n#     AUG_FLIP_UD = None\n#     AUG_ROT90 = None\n    \n    AUG_FLIP_LR = True\n    AUG_FLIP_UD = True\n    AUG_ROT90 = True\n\n#     ALIGN = None\n    # ALIGN = {'type': 'ds', 'x': 0.5, 'y': 0.5}\n    # ALIGN = {'type': 'rt', 'x': 0.5, 'y': 0.5}\n    ALIGN = {'type': 'ds', 'x': 0.408, 'y': 0.453}\n    AUG_DROPOUT = None\n#     AUG_DROPOUT = {'type': 'dropout', 'rate': 0.2}\n    # AUG_COARSE_DROPOUT = None\n    AUG_COARSE_DROPOUT = {'size': 256, 'threshold': 0.1}\n    BACKBONES = [\n#             'EfficientNetV2B0', \n            # 'EfficientNetV2B1',\n            # 'EfficientNetV2B2', \n            # 'EfficientNetV2B3', \n            # 'EfficientNetV2S', \n            # 'EfficientNetV2M',\n            'EfficientNetV2L',\n#             'EfficientNetB0',\n            # 'EfficientNetB1',\n            # 'EfficientNetB2',\n            # 'EfficientNetB3',\n            # 'EfficientNetB4',\n            # 'EfficientNetB5',\n            # 'EfficientNetB6',\n            # 'EfficientNetB7',\n#             'ResNet50',\n#             'ResNet101',\n#             'ResNet152',\n        ],\n    BACKBONE_CHANNEL = 3\n    WBC_WEIGHT = 10\n    LOSS_AUX = 0.05\n    \n    EPOCHS = 75\n    SAVE = True\n    SEED = 23\n    FOLD_SPLITS = 5\n    \n    LOGGING = False\n    \ncfg = CFG()","metadata":{"papermill":{"duration":0.026232,"end_time":"2023-06-08T13:14:52.348005","exception":false,"start_time":"2023-06-08T13:14:52.321773","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-19T15:10:06.91073Z","iopub.execute_input":"2023-07-19T15:10:06.911114Z","iopub.status.idle":"2023-07-19T15:10:06.921921Z","shell.execute_reply.started":"2023-07-19T15:10:06.911082Z","shell.execute_reply":"2023-07-19T15:10:06.921029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport gc\nimport re\nimport json\nimport time\nimport math\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random as rn\nfrom glob import glob\nfrom tqdm import tqdm\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedKFold\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\n\n# import icrgw","metadata":{"papermill":{"duration":45.53379,"end_time":"2023-06-08T13:15:37.91985","exception":false,"start_time":"2023-06-08T13:14:52.38606","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-19T15:10:06.923476Z","iopub.execute_input":"2023-07-19T15:10:06.923795Z","iopub.status.idle":"2023-07-19T15:10:50.076781Z","shell.execute_reply.started":"2023-07-19T15:10:06.923767Z","shell.execute_reply":"2023-07-19T15:10:50.075684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tf.__version__)","metadata":{"papermill":{"duration":0.014124,"end_time":"2023-06-08T13:15:37.938775","exception":false,"start_time":"2023-06-08T13:15:37.924651","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-19T15:10:50.078097Z","iopub.execute_input":"2023-07-19T15:10:50.078618Z","iopub.status.idle":"2023-07-19T15:10:50.083779Z","shell.execute_reply.started":"2023-07-19T15:10:50.078586Z","shell.execute_reply":"2023-07-19T15:10:50.082949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/input/icrgw-ds-utils/icrgw.py ./\nimport icrgw","metadata":{"papermill":{"duration":10.020854,"end_time":"2023-06-08T13:15:47.964217","exception":false,"start_time":"2023-06-08T13:15:37.943363","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-19T15:10:50.085979Z","iopub.execute_input":"2023-07-19T15:10:50.086282Z","iopub.status.idle":"2023-07-19T15:10:59.015543Z","shell.execute_reply.started":"2023-07-19T15:10:50.086254Z","shell.execute_reply":"2023-07-19T15:10:59.014472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import importlib\n# importlib.reload(icrgw)\n\nicrgw.init(cfg)","metadata":{"papermill":{"duration":0.336761,"end_time":"2023-06-08T13:15:48.308812","exception":false,"start_time":"2023-06-08T13:15:47.972051","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-19T15:10:59.016976Z","iopub.execute_input":"2023-07-19T15:10:59.017301Z","iopub.status.idle":"2023-07-19T15:10:59.058263Z","shell.execute_reply.started":"2023-07-19T15:10:59.017272Z","shell.execute_reply":"2023-07-19T15:10:59.057241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import importlib\n# importlib.reload(icrgw)\nicrgw.cfg = cfg","metadata":{"papermill":{"duration":0.017286,"end_time":"2023-06-08T13:15:48.334896","exception":false,"start_time":"2023-06-08T13:15:48.31761","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-19T15:10:59.059518Z","iopub.execute_input":"2023-07-19T15:10:59.059837Z","iopub.status.idle":"2023-07-19T15:10:59.064435Z","shell.execute_reply.started":"2023-07-19T15:10:59.059808Z","shell.execute_reply":"2023-07-19T15:10:59.063499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model(backbone_name):\n    xs_input = tf.keras.layers.Input((None, None, cfg.BACKBONE_CHANNEL))\n    encoder_filters=(16, 32, 64, 128, 256)\n    decoder_filters=(256, 128, 64, 32, 16)\n    decoder_block = icrgw.DecoderUpsamplingX2Block\n#     decoder_block = DecoderTransposeX2Block\n\n    bs = icrgw.create_backbone(backbone_name, cfg.BACKBONE_CHANNEL)\n\n    xs = xs_input\n    ss = []\n\n    for i in range(len(bs)):\n        xs = bs[i](xs)\n        ss.append(xs)\n\n    xs = ss[-1]\n    skips = ss[:-1][::-1]\n    \n    for i in range(5):\n        if i < len(skips):\n            skip = skips[i]\n        else:\n            skip = None\n        xs = decoder_block(decoder_filters[i], stage=i, use_batchnorm=True)(xs, skip)\n        \n    # model head (define number of output classes)\n    xs = tf.keras.layers.Conv2D(\n        filters=1,\n        kernel_size=(3, 3),\n        padding='same',\n        use_bias=True,\n        kernel_initializer='glorot_uniform',\n        name='final_conv',\n    )(xs)\n    ys = tf.keras.layers.Activation('sigmoid', name='unet_activation')(xs)\n\n    # create keras model instance\n    model = tf.keras.models.Model(xs_input, ys)\n\n    return model","metadata":{"papermill":{"duration":0.023127,"end_time":"2023-06-08T13:15:48.367207","exception":false,"start_time":"2023-06-08T13:15:48.34408","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-19T15:10:59.065688Z","iopub.execute_input":"2023-07-19T15:10:59.065989Z","iopub.status.idle":"2023-07-19T15:10:59.078934Z","shell.execute_reply.started":"2023-07-19T15:10:59.065962Z","shell.execute_reply":"2023-07-19T15:10:59.078014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with icrgw.strategy.scope():\n    train_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32)\n    train_accuracy = tf.keras.metrics.Mean('train_accuracy', dtype=tf.float32)\n    \n    train_i = tf.keras.metrics.Mean('train_i', dtype=tf.float32)    \n    train_u = tf.keras.metrics.Mean('train_u', dtype=tf.float32) \n    \n    valid_loss = tf.keras.metrics.Mean('valid_loss', dtype=tf.float32)\n    valid_accuracy = tf.keras.metrics.Mean('valid_accuracy', dtype=tf.float32)\n    \n    valid_i = tf.keras.metrics.Mean('valid_i', dtype=tf.float32)    \n    valid_u = tf.keras.metrics.Mean('valid_u', dtype=tf.float32) ","metadata":{"papermill":{"duration":0.240873,"end_time":"2023-06-08T13:15:48.615605","exception":false,"start_time":"2023-06-08T13:15:48.374732","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-19T15:10:59.080018Z","iopub.execute_input":"2023-07-19T15:10:59.080375Z","iopub.status.idle":"2023-07-19T15:10:59.314822Z","shell.execute_reply.started":"2023-07-19T15:10:59.080349Z","shell.execute_reply":"2023-07-19T15:10:59.313838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset_dis, valid_dataset_dis, train_steps_per_epoch, valid_steps_per_epoch = icrgw.get_ds()\ntrain_iterator = iter(train_dataset_dis)","metadata":{"papermill":{"duration":0.66567,"end_time":"2023-06-08T13:15:49.29014","exception":false,"start_time":"2023-06-08T13:15:48.62447","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-19T15:10:59.316151Z","iopub.execute_input":"2023-07-19T15:10:59.31652Z","iopub.status.idle":"2023-07-19T15:11:01.015742Z","shell.execute_reply.started":"2023-07-19T15:10:59.316489Z","shell.execute_reply":"2023-07-19T15:11:01.014516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def binary_focal_loss(gt, pr, gamma=2.0, alpha=0.25, **kwargs):\n    r\"\"\"Implementation of Focal Loss from the paper in binary classification\n\n    Formula:\n        loss = - gt * alpha * ((1 - pr)^gamma) * log(pr) \\\n               - (1 - gt) * alpha * (pr^gamma) * log(1 - pr)\n\n    Args:\n        gt: ground truth 4D keras tensor (B, H, W, C) or (B, C, H, W)\n        pr: prediction 4D keras tensor (B, H, W, C) or (B, C, H, W)\n        alpha: the same as weighting factor in balanced cross entropy, default 0.25\n        gamma: focusing parameter for modulating factor (1-p), default 2.0\n\n    \"\"\"\n    backend = K\n\n    # clip to prevent NaN's and Inf's\n    pr = backend.clip(pr, backend.epsilon(), 1.0 - backend.epsilon())\n\n    loss_1 = - gt * (alpha * backend.pow((1 - pr), gamma) * backend.log(pr))\n    loss_0 = - (1 - gt) * ((1 - alpha) * backend.pow((pr), gamma) * backend.log(1 - pr))\n#     loss = backend.mean(loss_0 + loss_1, axis=[1,2,3], keepdims=False)\n    loss = backend.mean(loss_0 + loss_1)\n    return loss\n\ndef focal_dice_loss(y_true, y_pred):\n    print(f'focal_dice_loss')\n    loss_1 = binary_focal_loss(y_true, y_pred, alpha=0.75) * 50\n    print(loss_1)\n    loss_2 = 1 - icrgw.f_score(y_true, y_pred)\n    print(loss_2)\n    loss = loss_1 + loss_2\n#     loss = loss_1\n    loss = tf.expand_dims(loss, 0)\n    return loss","metadata":{"papermill":{"duration":0.018195,"end_time":"2023-06-08T13:15:49.316382","exception":false,"start_time":"2023-06-08T13:15:49.298187","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for bb in cfg.BACKBONES[0]:\n\n    with icrgw.strategy.scope():\n        model = create_model(bb)\n#         optimizer = tf.keras.optimizers.Adamax()\n        optimizer = icrgw.Lion(1e-5)\n        model.load_weights('/kaggle/input/icrgw-train-single-ash-768-512-efv2l-0617/EfficientNetV2L.h5')\n\n    train_step, valid_step = icrgw.build(model, optimizer, focal_dice_loss,\n            train_loss, train_accuracy, train_i, train_u,\n            valid_loss, valid_accuracy, valid_i, valid_u)\n\n    lr_cb = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_score', mode='max', factor=0.2, patience=4, verbose=1, min_lr=1e-8)\n    model.optimizer = optimizer\n    lr_cb.set_model(model)\n\n    target_epochs = cfg.EPOCHS\n\n#     if icrgw.strategy.num_replicas_in_sync == 1:\n#         train_steps_per_epoch = 1\n#     valid_step(valid_dataset_dis)\n\n    for epoch in range(target_epochs):\n        print('\\nEpoch: {}/{}'.format(epoch, target_epochs))\n        start = time.time()\n\n        train_step(train_iterator, train_steps_per_epoch)\n        print(f'train loss: {train_loss.result():.4f}, \\\n            accuracy: {train_accuracy.result():.4f}, \\\n            score: {(2 * train_i.result()/train_u.result()).numpy()}'\n            )\n        train_loss.reset_states()\n        train_accuracy.reset_states()\n        train_i.reset_states()\n        train_u.reset_states()\n        \n        if epoch > -1:\n            valid_loss.reset_states()\n            valid_accuracy.reset_states()\n            valid_i.reset_states()\n            valid_u.reset_states()\n        #     for data in valid_dataset_dis:\n        #         valid_step(data)\n            valid_step(valid_dataset_dis)\n            score = (2 * valid_i.result()/valid_u.result()).numpy()\n            print(f'\\r\\r valid loss: {valid_loss.result():.4f}, \\\n                valid accuracy: {valid_accuracy.result():.4f}, \\\n                score: {score}',\n                )\n\n            lr_cb.on_epoch_end(epoch, {'val_score': score})\n        \n        elapsed = time.time() - start\n        print(f'{elapsed:.2f} S')\n        \n        if optimizer.lr < 1e-7:\n            break\n            \n    if cfg.SAVE: \n        if cfg.FLOAT_16:\n            model.save('m_16.h5')\n            policy = tf.keras.mixed_precision.Policy('float32')\n            tf.keras.mixed_precision.set_global_policy(policy)\n            mt = create_model(bb)\n            mt.load_weights('m_16.h5')\n            mt.save(f'{bb}.h5')\n            if icrgw.tpu:\n                policy = tf.keras.mixed_precision.Policy('mixed_bfloat16')\n                tf.keras.mixed_precision.set_global_policy(policy)\n            else:\n                policy = tf.keras.mixed_precision.Policy('mixed_float16')\n                tf.keras.mixed_precision.set_global_policy(policy)\n        else:\n            model.save(f'{bb}.h5')","metadata":{"papermill":{"duration":1172.154483,"end_time":"2023-06-08T13:35:21.478765","exception":false,"start_time":"2023-06-08T13:15:49.324282","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-19T15:11:01.0306Z","iopub.execute_input":"2023-07-19T15:11:01.030919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}