{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"ENV = 'kaggle'\n\nclass CFG:\n    DS_TYPE_SINGLE = 'SINGLE'\n    DS_TYPE_MULTIPLE = 'MULTIPLE'\n    DS_TYPE_SINGLE_ASH = 'SINGLE_ASH'\n    DS_TYPE_MULTIPLE_ASHS = 'MULTIPLE_ASHS'\n\n    DS_TYPE = DS_TYPE_SINGLE_ASH\n    FLOAT_16 = True\n    BATCH_SIZE = 16\n    ORI_SIZE = 256\n    TRAIN_SIZE = 512\n    VALID_SIZE = 768\n    AUG_ROT = 0\n    AUG_JPEG = None\n    AUG_COLOR = None\n    AUG_NOISE = None\n    AUG_FLIP_LR = {'shift_kernel': [\n        [0., 0., 0.],\n        [0., 1., 0.],\n        [0., 0., 0.],\n    ]}\n    AUG_FLIP_UD = {'shift_kernel': [\n        [0., 0., 0.],\n        [0., 1., 0.],\n        [0., 0., 0.],\n    ]}\n    AUG_ROT90 = {'shift_kernel': [\n        [0., 0., 0.],\n        [0., 1., 0.],\n        [0., 0., 0.],\n    ]}\n    ALIGN_KERNEL = None\n    ALIGN_LR = [\n        [0., 0., 0.],\n        [0., 1., 0.],\n        [0., 0., 0.],\n    ]\n    ALIGN_UD = [\n        [0., 0., 0.],\n        [0., 1., 0.],\n        [0., 0., 0.],\n    ]\n    AUG_DROPOUT = None\n    BACKBONES = [\n#             'EfficientNetV2B0', \n            # 'EfficientNetV2B1',\n            # 'EfficientNetV2B2', \n            # 'EfficientNetV2B3', \n            # 'EfficientNetV2S', \n            # 'EfficientNetV2M',\n            'EfficientNetV2L',\n#             'EfficientNetB0',\n            # 'EfficientNetB1',\n            # 'EfficientNetB2',\n            # 'EfficientNetB3',\n            # 'EfficientNetB4',\n            # 'EfficientNetB5',\n            # 'EfficientNetB6',\n            # 'EfficientNetB7',\n#             'ResNet50',\n#             'ResNet101',\n#             'ResNet152',\n        ],\n    BACKBONE_CHANNEL = 3\n    \n    EPOCHS = 100\n    SAVE = True\n    SEED = 23\n    FOLD_SPLITS = 5\n    \n    LOGGING = False\n    \ncfg = CFG()","metadata":{"papermill":{"duration":0.026232,"end_time":"2023-06-08T13:14:52.348005","exception":false,"start_time":"2023-06-08T13:14:52.321773","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-16T16:06:51.57116Z","iopub.execute_input":"2023-06-16T16:06:51.571464Z","iopub.status.idle":"2023-06-16T16:06:51.608183Z","shell.execute_reply.started":"2023-06-16T16:06:51.571435Z","shell.execute_reply":"2023-06-16T16:06:51.607356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport gc\nimport re\nimport json\nimport time\nimport math\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random as rn\nfrom glob import glob\nfrom tqdm import tqdm\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedKFold\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\n\n# import icrgw","metadata":{"papermill":{"duration":45.53379,"end_time":"2023-06-08T13:15:37.91985","exception":false,"start_time":"2023-06-08T13:14:52.38606","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-16T16:06:57.463007Z","iopub.execute_input":"2023-06-16T16:06:57.463712Z","iopub.status.idle":"2023-06-16T16:07:05.703451Z","shell.execute_reply.started":"2023-06-16T16:06:57.463675Z","shell.execute_reply":"2023-06-16T16:07:05.702414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tf.__version__)","metadata":{"papermill":{"duration":0.014124,"end_time":"2023-06-08T13:15:37.938775","exception":false,"start_time":"2023-06-08T13:15:37.924651","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-16T16:07:05.705482Z","iopub.execute_input":"2023-06-16T16:07:05.706284Z","iopub.status.idle":"2023-06-16T16:07:05.713235Z","shell.execute_reply.started":"2023-06-16T16:07:05.706248Z","shell.execute_reply":"2023-06-16T16:07:05.711244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/input/icrgw-ds-utils/icrgw.py ./\nimport icrgw","metadata":{"papermill":{"duration":10.020854,"end_time":"2023-06-08T13:15:47.964217","exception":false,"start_time":"2023-06-08T13:15:37.943363","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-16T16:07:05.714523Z","iopub.execute_input":"2023-06-16T16:07:05.715163Z","iopub.status.idle":"2023-06-16T16:07:06.742208Z","shell.execute_reply.started":"2023-06-16T16:07:05.715125Z","shell.execute_reply":"2023-06-16T16:07:06.741026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import importlib\n# importlib.reload(icrgw)\n\nicrgw.init(cfg)","metadata":{"papermill":{"duration":0.336761,"end_time":"2023-06-08T13:15:48.308812","exception":false,"start_time":"2023-06-08T13:15:47.972051","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-16T16:07:06.752429Z","iopub.execute_input":"2023-06-16T16:07:06.753064Z","iopub.status.idle":"2023-06-16T16:07:09.992639Z","shell.execute_reply.started":"2023-06-16T16:07:06.753032Z","shell.execute_reply":"2023-06-16T16:07:09.991611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import importlib\n# importlib.reload(icrgw)\nicrgw.cfg = cfg","metadata":{"papermill":{"duration":0.017286,"end_time":"2023-06-08T13:15:48.334896","exception":false,"start_time":"2023-06-08T13:15:48.31761","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-16T16:07:42.647605Z","iopub.execute_input":"2023-06-16T16:07:42.648684Z","iopub.status.idle":"2023-06-16T16:07:42.653607Z","shell.execute_reply.started":"2023-06-16T16:07:42.648632Z","shell.execute_reply":"2023-06-16T16:07:42.652662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model(backbone_name):\n    xs_input = tf.keras.layers.Input((None, None, cfg.BACKBONE_CHANNEL))\n    encoder_filters=(16, 32, 64, 128, 256)\n    decoder_filters=(256, 128, 64, 32, 16)\n    decoder_block = icrgw.DecoderUpsamplingX2Block\n#     decoder_block = DecoderTransposeX2Block\n\n    bs = icrgw.create_backbone(backbone_name, cfg.BACKBONE_CHANNEL)\n\n    xs = xs_input\n    ss = []\n\n    for i in range(len(bs)):\n        xs = bs[i](xs)\n        ss.append(xs)\n\n    xs = ss[-1]\n    skips = ss[:-1][::-1]\n    \n    for i in range(5):\n        if i < len(skips):\n            skip = skips[i]\n        else:\n            skip = None\n        xs = decoder_block(decoder_filters[i], stage=i, use_batchnorm=True)(xs, skip)\n        \n    # model head (define number of output classes)\n    xs = tf.keras.layers.Conv2D(\n        filters=1,\n        kernel_size=(3, 3),\n        padding='same',\n        use_bias=True,\n        kernel_initializer='glorot_uniform',\n        name='final_conv',\n    )(xs)\n    ys = tf.keras.layers.Activation('sigmoid', name='unet_activation')(xs)\n\n    # create keras model instance\n    model = tf.keras.models.Model(xs_input, ys)\n\n    return model","metadata":{"papermill":{"duration":0.023127,"end_time":"2023-06-08T13:15:48.367207","exception":false,"start_time":"2023-06-08T13:15:48.34408","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-16T16:07:43.078116Z","iopub.execute_input":"2023-06-16T16:07:43.078491Z","iopub.status.idle":"2023-06-16T16:07:43.090222Z","shell.execute_reply.started":"2023-06-16T16:07:43.07846Z","shell.execute_reply":"2023-06-16T16:07:43.088514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with icrgw.strategy.scope():\n    train_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32)\n    train_accuracy = tf.keras.metrics.Mean('train_accuracy', dtype=tf.float32)\n    \n    train_i = tf.keras.metrics.Mean('train_i', dtype=tf.float32)    \n    train_u = tf.keras.metrics.Mean('train_u', dtype=tf.float32) \n    \n    valid_loss = tf.keras.metrics.Mean('valid_loss', dtype=tf.float32)\n    valid_accuracy = tf.keras.metrics.Mean('valid_accuracy', dtype=tf.float32)\n    \n    valid_i = tf.keras.metrics.Mean('valid_i', dtype=tf.float32)    \n    valid_u = tf.keras.metrics.Mean('valid_u', dtype=tf.float32) ","metadata":{"papermill":{"duration":0.240873,"end_time":"2023-06-08T13:15:48.615605","exception":false,"start_time":"2023-06-08T13:15:48.374732","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-16T16:07:43.460378Z","iopub.execute_input":"2023-06-16T16:07:43.461399Z","iopub.status.idle":"2023-06-16T16:07:43.504017Z","shell.execute_reply.started":"2023-06-16T16:07:43.461364Z","shell.execute_reply":"2023-06-16T16:07:43.503077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset_dis, valid_dataset_dis, train_steps_per_epoch, valid_steps_per_epoch = icrgw.get_ds()\ntrain_iterator = iter(train_dataset_dis)","metadata":{"papermill":{"duration":0.66567,"end_time":"2023-06-08T13:15:49.29014","exception":false,"start_time":"2023-06-08T13:15:48.62447","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-16T16:07:43.898244Z","iopub.execute_input":"2023-06-16T16:07:43.899278Z","iopub.status.idle":"2023-06-16T16:07:43.964756Z","shell.execute_reply.started":"2023-06-16T16:07:43.899237Z","shell.execute_reply":"2023-06-16T16:07:43.963759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bce_dice_loss(y_true, y_pred):\n    print('customized bce_dice_loss')\n    weights = ((y_true * 10.) + 1.)[:,:,:,0]\n    loss = tf.keras.losses.binary_crossentropy(y_true, y_pred)\n    loss = tf.reduce_mean(loss * weights, axis=[1,2])\n    \n    loss = loss + (1 - icrgw.f_score(y_true, y_pred))\n    return loss\n\nicrgw.bce_dice_loss = bce_dice_loss","metadata":{"papermill":{"duration":0.018195,"end_time":"2023-06-08T13:15:49.316382","exception":false,"start_time":"2023-06-08T13:15:49.298187","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-16T16:07:44.289748Z","iopub.execute_input":"2023-06-16T16:07:44.290297Z","iopub.status.idle":"2023-06-16T16:07:44.297919Z","shell.execute_reply.started":"2023-06-16T16:07:44.290258Z","shell.execute_reply":"2023-06-16T16:07:44.296645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for bb in cfg.BACKBONES[0]:\n\n    with icrgw.strategy.scope():\n        model = create_model(bb)\n#         optimizer = tf.keras.optimizers.Adamax()\n        optimizer = icrgw.Lion()\n\n    train_step, valid_step = icrgw.build(model, optimizer, icrgw.bce_dice_loss,\n            train_loss, train_accuracy, train_i, train_u,\n            valid_loss, valid_accuracy, valid_i, valid_u)\n\n    lr_cb = tf.keras.callbacks.ReduceLROnPlateau(factor=0.2, patience=6, verbose=1, min_lr=1e-8)\n    model.optimizer = optimizer\n    lr_cb.set_model(model)\n\n    target_epochs = cfg.EPOCHS\n\n#     if icrgw.strategy.num_replicas_in_sync == 1:\n#         train_steps_per_epoch = 1\n#     valid_step(valid_dataset_dis)\n\n    for epoch in range(target_epochs):\n        print('\\nEpoch: {}/{}'.format(epoch, target_epochs))\n        start = time.time()\n\n        train_step(train_iterator, train_steps_per_epoch)\n        print(f'train loss: {train_loss.result():.4f}, \\\n            accuracy: {train_accuracy.result():.4f}, \\\n            score: {(2 * train_i.result()/train_u.result()).numpy()}'\n            )\n        train_loss.reset_states()\n        train_accuracy.reset_states()\n        train_i.reset_states()\n        train_u.reset_states()\n        \n        if epoch > 5:\n            valid_loss.reset_states()\n            valid_accuracy.reset_states()\n            valid_i.reset_states()\n            valid_u.reset_states()\n        #     for data in valid_dataset_dis:\n        #         valid_step(data)\n            valid_step(valid_dataset_dis)\n            print(f'\\r\\r valid loss: {valid_loss.result():.4f}, \\\n                valid accuracy: {valid_accuracy.result():.4f}, \\\n                score: {(2 * valid_i.result()/valid_u.result()).numpy()}',\n                )\n\n            lr_cb.on_epoch_end(epoch, {'val_loss': valid_loss.result()})\n        \n        elapsed = time.time() - start\n        print(f'{elapsed:.2f} S')\n        \n        if optimizer.lr < 1e-7:\n            break\n            \n    if cfg.SAVE: \n        if cfg.FLOAT_16:\n            model.save('m_16.h5')\n            policy = tf.keras.mixed_precision.Policy('float32')\n            tf.keras.mixed_precision.set_global_policy(policy)\n            mt = create_model(bb)\n            mt.load_weights('m_16.h5')\n            mt.save(f'{bb}.h5')\n            if icrgw.tpu:\n                policy = tf.keras.mixed_precision.Policy('mixed_bfloat16')\n                tf.keras.mixed_precision.set_global_policy(policy)\n            else:\n                policy = tf.keras.mixed_precision.Policy('mixed_float16')\n                tf.keras.mixed_precision.set_global_policy(policy)\n        else:\n            model.save(f'{bb}.h5')","metadata":{"papermill":{"duration":1172.154483,"end_time":"2023-06-08T13:35:21.478765","exception":false,"start_time":"2023-06-08T13:15:49.324282","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}