{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hello fellow Kagglers,\n\nThis notebook demonstrates the training process on a TPU in Tensorflow.\n\nThanks to the use of a [TPU (Tensor Processing Unit)](https://cloud.google.com/tpu) training takes about an hour.\n\nThe TFREcord dataset contains cropped images sized 1344x768, created in [this notebook](https://www.kaggle.com/code/markwijkhuizen/rsna-preprocessing-tfrecords-640x512-dataset).\n\n20% of the data is used for validation, which reaches ~0.20 pF1 with the best threshold.\n\n**Things that did not work for me:**\n\n* [SigmoidFocalCrossEntropy](https://www.tensorflow.org/addons/api_docs/python/tfa/losses/SigmoidFocalCrossEntropy)\n* Increasing model size to for example EfficientNetV2S\n\n**Things that did work for me:**\n\n* Class weights: give minority class weight of 10\n* Training on TPU instead of GPU: larger batch size (16x2->16x8) giving larger probability of having positive sample in batch\n* Cropping Images\n* Using Cropped Image Ratio\n\nI enjoy this competition and will update this notebook frequently, stay tuned!\n\n**V2**\n\n* Cropped images in 1344x768 resolution\n* EfficientNetV2T\n* Added augmentations\n* Single image modal instead of both CC and MLO views as input\n\n**V3**\n\n* Switch to modern ConvNextV2 models [paper: ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/pdf/2301.00808.pdf)\n* Reduced learning rate 1e-4xN_REPLICAS -> 5e-6xN_REPLICAS\n* Adam -> AdamW optimizer with 0.01 Weight Decay ratio\n* Removed Warmup Epochs\n\n[RSNA Cropped TFRecords 768x1344 Dataset](https://www.kaggle.com/code/markwijkhuizen/rsna-cropped-tfrecords-768x1344-dataset)\n\n[Inference Notebook](https://www.kaggle.com/markwijkhuizen/rsna-efficientnetv2-inference-tensorflow)","metadata":{"papermill":{"duration":0.052016,"end_time":"2023-02-17T04:02:31.635053","exception":false,"start_time":"2023-02-17T04:02:31.583037","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Install ConvNextV2 Models From Keras CV Attention Models Pip Package\n!pip install -q --upgrade pip\n\n!pip install -q /lib/wheels/tensorflow-2.9.1-cp38-cp38-linux_x86_64.whl\n!pip install -q tensorflow-addons==0.18.0\n!pip install -q opencv-python-headless\n\n!pip install --no-deps /kaggle/input/keras-cv-attention-models/keras_cv_attention_models-1.3.9-py3-none-any.whl\n!pip install -qU scikit-learn\n!pip install -q seaborn","metadata":{"papermill":{"duration":77.955926,"end_time":"2023-02-17T04:03:49.607604","exception":false,"start_time":"2023-02-17T04:02:31.651678","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\n\nfrom multiprocessing import cpu_count\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom keras_cv_attention_models import convnext\n\nimport os\nimport time\nimport pickle\nimport math\nimport random\nimport sys\nimport cv2\nimport gc\nimport datetime\n\nprint(f'Tensorflow Version: {tf.__version__}')\nprint(f'Python Version: {sys.version}')","metadata":{"papermill":{"duration":18.654733,"end_time":"2023-02-17T04:04:08.279891","exception":false,"start_time":"2023-02-17T04:03:49.625158","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"now = datetime.datetime.now().strftime(\"%d-%b-%Y %H-%M-%S\")\nnp.save(now, np.array([now]))","metadata":{"papermill":{"duration":0.030783,"end_time":"2023-02-17T04:04:08.332447","exception":false,"start_time":"2023-02-17T04:04:08.301664","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mixed Precision Policy","metadata":{"papermill":{"duration":0.020071,"end_time":"2023-02-17T04:04:08.373174","exception":false,"start_time":"2023-02-17T04:04:08.353103","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# float32 or mixed_float16 (mixed precision: compute float16, variable float32)\n# TPU is fast enough and has enough memory to use float32\npolicy = tf.keras.mixed_precision.Policy('float32')\ntf.keras.mixed_precision.set_global_policy(policy)\n\nprint(f'Compute dtype: {tf.keras.mixed_precision.global_policy().compute_dtype}')\nprint(f'Variable dtype: {tf.keras.mixed_precision.global_policy().variable_dtype}')","metadata":{"papermill":{"duration":0.030395,"end_time":"2023-02-17T04:04:08.423671","exception":false,"start_time":"2023-02-17T04:04:08.393276","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Matplotlib Config","metadata":{"papermill":{"duration":0.019798,"end_time":"2023-02-17T04:04:08.464326","exception":false,"start_time":"2023-02-17T04:04:08.444528","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"papermill":{"duration":0.03059,"end_time":"2023-02-17T04:04:08.514806","exception":false,"start_time":"2023-02-17T04:04:08.484216","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{"papermill":{"duration":0.020415,"end_time":"2023-02-17T04:04:08.555507","exception":false,"start_time":"2023-02-17T04:04:08.535092","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    TPU = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=\"local\")  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', TPU.master())\nexcept ValueError:\n    print('Running on GPU')\n    TPU = None\n\nif TPU:\n    IS_TPU = True\n    tf.config.experimental_connect_to_cluster(TPU)\n    tf.tpu.experimental.initialize_tpu_system(TPU)\n    STRATEGY = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n    IS_TPU = False\n    STRATEGY = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nN_REPLICAS = STRATEGY.num_replicas_in_sync\nprint(f'N_REPLICAS: {N_REPLICAS}, IS_TPU: {IS_TPU}')","metadata":{"_kg_hide-output":true,"papermill":{"duration":15.533452,"end_time":"2023-02-17T04:04:24.109006","exception":false,"start_time":"2023-02-17T04:04:08.575554","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For TPU's the dataset needs to be stored in Google Cloud\n# Retrieve the Google Cloud location of the dataset\n# GCS_DS_PATH = KaggleDatasets().get_gcs_path('rsna-preprocessing-tfrecords-640x512-dataset-pub')\nGCS_DS_PATH = '/kaggle/input/rsnabcd-cropped-png-768x1344-datase/kaggle/train_images'","metadata":{"papermill":{"duration":0.032491,"end_time":"2023-02-17T04:04:24.165324","exception":false,"start_time":"2023-02-17T04:04:24.132833","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 43\nDEBUG = False\n\n# Image dimensions\nIMG_HEIGHT = 1344\nIMG_WIDTH = 768\nN_CHANNELS = 1\nINPUT_SHAPE = (IMG_HEIGHT, IMG_WIDTH, 1)\nN_SAMPLES_TFRECORDS = 550\n\nN_SPLITS = 20\nUPSAMPLE = 5\nfold = 0 \n\n# Peak Learning Rate\nLR_MAX = 5e-6 * N_REPLICAS\nWD_RATIO = 0.01\n\nN_WARMUP_EPOCHS = 0\nN_EPOCHS = 9\n\n# Batch size\nBATCH_SIZE = 16 * N_REPLICAS\n\n# Is Interactive Flag and COrresponding Verbosity Method\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\nVERBOSE = 1 if IS_INTERACTIVE else 2\n\n# Tensorflow AUTO flag\nAUTO = tf.data.experimental.AUTOTUNE\n\nprint(f'BATCH_SIZE: {BATCH_SIZE}')","metadata":{"papermill":{"duration":0.035798,"end_time":"2023-02-17T04:04:24.22466","exception":false,"start_time":"2023-02-17T04:04:24.188862","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seed","metadata":{"papermill":{"duration":0.023124,"end_time":"2023-02-17T04:04:24.27159","exception":false,"start_time":"2023-02-17T04:04:24.248466","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Seed all random number generators\ndef seed_everything(seed=SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\nseed_everything()","metadata":{"papermill":{"duration":0.031499,"end_time":"2023-02-17T04:04:24.327149","exception":false,"start_time":"2023-02-17T04:04:24.29565","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{"papermill":{"duration":0.021506,"end_time":"2023-02-17T04:04:24.370408","exception":false,"start_time":"2023-02-17T04:04:24.348902","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Train DataFrame\ntrain = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ntrain['file_name'] = GCS_DS_PATH+'/' + train['patient_id'].astype(str)+'/' + train['image_id'].astype(str) + '.png'\n\nnum_bins = 5\ntrain[\"age_bin\"] = pd.cut(train['age'].values.reshape(-1), bins=num_bins, labels=False)\n\nstrat_cols = [\n    'laterality', 'view', 'biopsy','invasive', 'BIRADS', 'age_bin',\n    'implant', 'density','machine_id', 'difficult_negative_case',\n    'cancer',\n]\ntrain['stratify'] = ''\nfor col in strat_cols:\n    train['stratify'] += train[col].astype(str)\n\nskf = StratifiedGroupKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\nfor fold, (train_idx, val_idx) in enumerate(skf.split(train, train['stratify'], train[\"patient_id\"])):\n    train.loc[val_idx, 'fold'] = fold\n\n\ndisplay(train.head())\ndisplay(train.info())","metadata":{"papermill":{"duration":25.929021,"end_time":"2023-02-17T04:04:50.320986","exception":false,"start_time":"2023-02-17T04:04:24.391965","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{"papermill":{"duration":0.024806,"end_time":"2023-02-17T04:04:50.36981","exception":false,"start_time":"2023-02-17T04:04:50.345004","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# short Tensorflow randin integer function\ndef tf_rand_int(minval, maxval, dtype=tf.int64):\n    minval = tf.cast(minval, dtype)\n    maxval = tf.cast(maxval, dtype)\n    return tf.random.uniform(shape=(), minval=minval, maxval=maxval, dtype=dtype)\n\n# chance of 1 in k\ndef one_in(k):\n    return 0 == tf_rand_int(0, k)","metadata":{"papermill":{"duration":0.036072,"end_time":"2023-02-17T04:04:50.430501","exception":false,"start_time":"2023-02-17T04:04:50.394429","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"papermill":{"duration":0.02371,"end_time":"2023-02-17T04:04:50.478814","exception":false,"start_time":"2023-02-17T04:04:50.455104","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Function to benchmark the dataset\ndef benchmark_dataset(dataset, num_epochs=3, n_steps_per_epoch=10, bs=BATCH_SIZE):\n    start_time = time.perf_counter()\n    for epoch_num in range(num_epochs):\n        for idx, (inputs, labels) in enumerate(dataset.take(n_steps_per_epoch + 1)):\n            if idx == 0:\n                epoch_start = time.perf_counter()\n            elif idx == 1 and epoch_num == 0:\n                image = inputs['image']\n                print(f'image shape: {image.shape}, labels shape: {labels.shape}, image dtype: {image.dtype}, labels dtype: {labels.dtype}')\n            else:\n                pass\n        \n        epoch_t = time.perf_counter() - epoch_start\n        mean_step_t = round(epoch_t / n_steps_per_epoch * 1000, 1)\n        n_imgs_per_s = int(1 / (mean_step_t / 1000) * bs)\n        print(f'epoch {epoch_num} took: {round(epoch_t, 2)} sec, mean step duration: {mean_step_t}ms, images/s: {n_imgs_per_s}')","metadata":{"papermill":{"duration":0.036089,"end_time":"2023-02-17T04:04:50.538356","exception":false,"start_time":"2023-02-17T04:04:50.502267","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plots a batch of images\ndef show_batch(dataset, n_rows=16, n_cols=4):\n    inputs, targets = next(iter(dataset))\n    images = inputs['image'].numpy().squeeze()\n    fig, axes = plt.subplots(nrows=n_rows, ncols=n_cols, figsize=(n_cols*4, n_rows*7))\n    for r in range(n_rows):\n        for c in range(n_cols):\n            idx = r * n_cols + c\n            # Image\n            img = images[idx]\n            axes[r, c].imshow(img)\n            # Target\n            target = targets[idx]\n            axes[r, c].set_title(f'target: {target}', fontsize=16, pad=5)\n        \n    plt.show()","metadata":{"papermill":{"duration":0.036812,"end_time":"2023-02-17T04:04:50.598932","exception":false,"start_time":"2023-02-17T04:04:50.56212","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Decodes the TFRecords\n# def decode_image(record_bytes):\n#     features = tf.io.parse_single_example(record_bytes, {\n#         'image': tf.io.FixedLenFeature([], tf.string),\n#         'target': tf.io.FixedLenFeature([], tf.int64),\n#         'patient_id': tf.io.FixedLenFeature([], tf.int64),\n#     })\n    \n#     # Decode PNG Image\n#     image = tf.io.decode_png(features['image'], channels=N_CHANNELS)\n#     # Explicit reshape needed for TPU\n#     image = tf.reshape(image, [IMG_HEIGHT, IMG_WIDTH, N_CHANNELS])\n\n#     target = features['target']\n    \n#     return { 'image': image }, target\n\ndef decode_image(filename, target):\n    # Decode PNG Image\n    bits = tf.io.read_file(filename)\n    image = tf.io.decode_png(bits, channels=N_CHANNELS)\n    \n    # Set target shape\n    image.set_shape((IMG_HEIGHT, IMG_WIDTH, N_CHANNELS))\n    \n    return { 'image': image }, target","metadata":{"papermill":{"duration":0.034997,"end_time":"2023-02-17T04:04:50.658985","exception":false,"start_time":"2023-02-17T04:04:50.623988","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_image(X, y):\n    image = X['image']\n    \n    # Random Brightness\n    image = tf.image.random_brightness(image, 0.10)\n    \n    # Random Contrast\n    image = tf.image.random_contrast(image, 0.90, 1.10)\n    \n    # Random JPEG Quality\n    image = tf.image.random_jpeg_quality(image, 85, 100)\n    \n    # Random crop image with maximum of 10%\n    ratio = tf.random.uniform([], 0.85, 1.00)\n    img_height_crop = tf.cast(ratio * IMG_HEIGHT, tf.int32)\n    img_width_crop = tf.cast(ratio * IMG_WIDTH, tf.int32)\n    # Random offset for crop\n    img_height_offset = tf_rand_int(0, IMG_HEIGHT - img_height_crop)\n    img_width_offset = 0\n    # Crop And Resize\n    image = tf.slice(image, [img_height_offset, img_width_offset, 0], [img_height_crop, img_width_crop, N_CHANNELS])\n    image = tf.image.resize(image, [IMG_HEIGHT, IMG_WIDTH], method=tf.image.ResizeMethod.BILINEAR)\n    # Clip pixel values in range [0,255] to prevent underflow/overflow\n    image = tf.clip_by_value(image, 0, 255)\n    image = tf.cast(image, tf.uint8)\n    \n    return { 'image': image }, y","metadata":{"papermill":{"duration":0.038725,"end_time":"2023-02-17T04:04:50.722328","exception":false,"start_time":"2023-02-17T04:04:50.683603","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Undersample majority class (0/negative) by randomly dropping them\n# def undersample_majority(X, y):\n#     # Filter 2/3 of negative samples to upsample positive samples by a factor 3\n#     return y == 1 or tf.random.uniform([]) > 0.76","metadata":{"papermill":{"duration":0.032931,"end_time":"2023-02-17T04:04:50.779976","exception":false,"start_time":"2023-02-17T04:04:50.747045","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # TFRecord file paths\n# TFRECORDS_FILE_PATHS = sorted(tf.io.gfile.glob(f'{GCS_DS_PATH}/*.tfrecords'))\n# print(f'Found {len(TFRECORDS_FILE_PATHS)} TFRecords')\n\n# # Train Test Split\n# TFRECORDS_TRAIN, TFRECORDS_VAL = train_test_split(TFRECORDS_FILE_PATHS, train_size=0.95, random_state=SEED, shuffle=True)\n# print(f'# TFRECORDS_TRAIN: {len(TFRECORDS_TRAIN)}, # TFRECORDS_VAL: {len(TFRECORDS_VAL)}')","metadata":{"papermill":{"duration":0.032546,"end_time":"2023-02-17T04:04:50.837326","exception":false,"start_time":"2023-02-17T04:04:50.80478","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(df, bs=BATCH_SIZE, val=False, debug=True):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    \n    # Initialize dataset with TFRecords\n    dataset = tf.data.Dataset.from_tensor_slices((df['file_name'], df['cancer']))\n    \n    if not val:\n        dataset = dataset.shuffle(len(df))\n    \n    # Decode mapping\n    dataset = dataset.map(decode_image, num_parallel_calls=AUTO)\n\n    if not val:\n#         dataset = dataset.filter(undersample_majority)\n        dataset = dataset.map(augment_image, num_parallel_calls=AUTO)\n        dataset = dataset.with_options(ignore_order)\n        dataset = dataset.repeat()\n\n    dataset = dataset.batch(bs, drop_remainder=not val)\n    dataset = dataset.prefetch(AUTO)\n    \n    return dataset","metadata":{"papermill":{"duration":0.035737,"end_time":"2023-02-17T04:04:50.897672","exception":false,"start_time":"2023-02-17T04:04:50.861935","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_f = train[(train['fold'] != fold) & (train['cancer'] == 0)]\n# df_t = train[(train['fold'] != fold) & (train['cancer'] == 1)]#.sample(frac=UPSAMPLE, replace=True, random_state=SEED)\n# df_train = pd.concat([df_f, df_t]).sample(frac=1, random_state=SEED).reset_index(drop=True)\n\n# df_f = train[(train['fold'] == fold) & (train['cancer'] == 0)]\n# df_t = train[(train['fold'] == fold) & (train['cancer'] == 1)]#.sample(frac=UPSAMPLE, replace=True, random_state=SEED)\n# df_test = pd.concat([df_f, df_t]).sample(frac=1, random_state=SEED).reset_index(drop=True)","metadata":{"papermill":{"duration":0.032438,"end_time":"2023-02-17T04:04:50.95479","exception":false,"start_time":"2023-02-17T04:04:50.922352","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Get Train/Validation datasets\n# train_dataset = get_dataset(df_train, val=False, debug=False)\n# val_dataset = get_dataset(df_test, val=True, debug=False)\n\n# TRAIN_STEPS_PER_EPOCH = len(df_train) // BATCH_SIZE\n# VAL_STEPS_PER_EPOCH = len(df_test) // BATCH_SIZE\n# print(f'TRAIN_STEPS_PER_EPOCH: {TRAIN_STEPS_PER_EPOCH}, VAL_STEPS_PER_EPOCH: {VAL_STEPS_PER_EPOCH}')","metadata":{"papermill":{"duration":0.034171,"end_time":"2023-02-17T04:04:51.014013","exception":false,"start_time":"2023-02-17T04:04:50.979842","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Sanity check, image and label statistics\n# X_batch, y_batch = next(iter(get_dataset(df_test, val=False)))\n# image = X_batch['image'].numpy()\n# print(f'image shape: {image.shape}, y_batch shape: {y_batch.shape}')\n# print(f'image dtype: {image.dtype}, y_batch dtype: {y_batch.dtype}')\n# print(f'image min: {image.min():.2f}, max: {image.max():.2f}')","metadata":{"papermill":{"duration":0.032714,"end_time":"2023-02-17T04:04:51.071647","exception":false,"start_time":"2023-02-17T04:04:51.038933","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Benchmark Dataset\n# benchmark_dataset(get_dataset(TFRECORDS_TRAIN, val=False))","metadata":{"papermill":{"duration":0.031595,"end_time":"2023-02-17T04:04:51.12783","exception":false,"start_time":"2023-02-17T04:04:51.096235","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show what we will be training on\nshow_batch(get_dataset(train, bs=BATCH_SIZE, val=False))","metadata":{"_kg_hide-output":true,"papermill":{"duration":16.552949,"end_time":"2023-02-17T04:05:07.705107","exception":false,"start_time":"2023-02-17T04:04:51.152158","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Imbalance","metadata":{"papermill":{"duration":0.105888,"end_time":"2023-02-17T04:05:07.920978","exception":false,"start_time":"2023-02-17T04:05:07.81509","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Label Distribution Train With Undersampled Majority Class\nN = 128\ntrain_labels = []\nfor _, labels in get_dataset(train, val=False).take(N):\n    train_labels += labels.numpy().tolist()\n    \ndisplay(pd.concat((\n        pd.Series(train_labels).value_counts(normalize=True).to_frame('Train Label Ratio'),\n        pd.Series(train_labels).value_counts().to_frame('Train Label Count'),\n    ), axis=1)\n)","metadata":{"papermill":{"duration":21.408027,"end_time":"2023-02-17T04:05:29.429756","exception":false,"start_time":"2023-02-17T04:05:08.021729","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# pF1 Metric\n\ninspiration: [RSNA-BCD: EfficientNet [TF][TPU-1VM][Train]](https://www.kaggle.com/code/awsaf49/rsna-bcd-efficientnet-tf-tpu-1vm-train#Metric)\n\nThe source implementation is however buggy, it is a moving average which does not reset each epoch. The implementation below does reset each epoch.","metadata":{"papermill":{"duration":0.10977,"end_time":"2023-02-17T04:05:29.644722","exception":false,"start_time":"2023-02-17T04:05:29.534952","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Tensorflow custom metric is just a conventional class object\nclass pF1(tf.keras.metrics.Metric):\n    # Initialize properties\n    def __init__(self, name='pF1', **kwargs):\n        super(pF1, self).__init__(name=name, **kwargs)\n        self.tc = self.add_weight(name='tc', initializer='zeros')\n        self.tp = self.add_weight(name='tp', initializer='zeros')\n        self.fp = self.add_weight(name='fp', initializer='zeros')\n\n    # Update state called on each batch with true and predicted labels\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        self.tc.assign_add(tf.cast(tf.reduce_sum(y_true), tf.float32))\n        self.tp.assign_add(tf.cast(tf.reduce_sum((y_pred[y_true == 1])), tf.float32))\n        self.fp.assign_add(tf.cast(tf.reduce_sum((y_pred[y_true == 0])), tf.float32))\n\n    # Result function is called to obtain result which is printed in progress bar\n    def result(self):\n        if self.tc == 0 or (self.tp + self.fp) == 0:\n            return 0.0\n        else:\n            precision = self.tp / (self.tp + self.fp)\n            recall = self.tp / (self.tc)\n            return 2 * (precision * recall) / (precision + recall)\n\n    # Reset state is called after each epoch to start fresh each epoch\n    def reset_state(self):\n        self.tc.assign(0)\n        self.tp.assign(0)\n        self.fp.assign(0)","metadata":{"papermill":{"duration":0.138626,"end_time":"2023-02-17T04:05:29.884311","exception":false,"start_time":"2023-02-17T04:05:29.745685","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GeMPoolingLayer(tf.keras.layers.Layer):\n    def __init__(self, p=3., train_p=False):\n        super().__init__()\n        if train_p:\n            self.p = tf.Variable(p, dtype=tf.float32)\n        else:\n            self.p = p\n        self.eps = 1e-6\n\n    def call(self, inputs: tf.Tensor, **kwargs):\n        inputs = tf.clip_by_value(inputs, clip_value_min=1e-6, clip_value_max=tf.reduce_max(inputs))\n        inputs = tf.pow(inputs, self.p)\n        inputs = tf.reduce_mean(inputs, axis=[1, 2], keepdims=False)\n        inputs = tf.pow(inputs, 1./self.p)\n        return inputs\n    \n    def compute_output_shape(self, input_shape):\n        return tuple([None, input_shape[-1]])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# source: https://www.kaggle.com/code/sohier/probabilistic-f-score\n# Competition Leaderboard Metric\ndef pfbeta(labels, predictions, beta=1):\n    y_true_count = 0\n    ctp = 0\n    cfp = 0\n\n    for idx in range(len(labels)):\n        prediction = min(max(predictions[idx], 0), 1)\n        if (labels[idx]):\n            y_true_count += 1\n            ctp += prediction\n        else:\n            cfp += prediction\n\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0\n\ndef pfbeta_thr(labels, preds, get_thr=False):\n    thrs = tf.range(0, 1, 0.05)\n    best_score = tf.constant(0, dtype=tf.float32)\n    best_thr = tf.constant(0, dtype=tf.float32)\n    \n    for thr in thrs:\n        score = tf.cast(pfbeta(labels, tf.cast(preds>thr, tf.float32)), tf.float32)\n        best_score = tf.cond(score > best_score, lambda: score, lambda: best_score)\n        best_thr = tf.cond(score == best_score, lambda: thr, lambda: best_thr)\n    \n    if get_thr:\n        return best_score, best_thr\n    return best_score","metadata":{"papermill":{"duration":0.138855,"end_time":"2023-02-17T04:05:30.131326","exception":false,"start_time":"2023-02-17T04:05:29.992471","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.108355,"end_time":"2023-02-17T04:05:30.571119","exception":false,"start_time":"2023-02-17T04:05:30.462764","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def normalize(image):\n    # Repeat channels to create 3 channel images required by pretrained EfficientNetV2 models\n    image = tf.repeat(image, repeats=3, axis=3)\n    # Cast to float 32\n    image = tf.cast(image, tf.float32)\n    # Normalize with respect to ImageNet mean/std\n    image = tf.keras.applications.imagenet_utils.preprocess_input(image, mode='torch')\n\n    return image","metadata":{"papermill":{"duration":0.124817,"end_time":"2023-02-17T04:05:30.79995","exception":false,"start_time":"2023-02-17T04:05:30.675133","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# abc = convnext.ConvNeXtV2Tiny(\n#             input_shape=(IMG_HEIGHT, IMG_WIDTH, 3),\n#             pretrained='imagenet21k-ft1k',\n#             num_classes=0,\n#         )","metadata":{"papermill":{"duration":0.109525,"end_time":"2023-02-17T04:05:31.01202","exception":false,"start_time":"2023-02-17T04:05:30.902495","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    # Verify Mixed Policy Settings\n    print(f'Compute dtype: {tf.keras.mixed_precision.global_policy().compute_dtype}')\n    print(f'Variable dtype: {tf.keras.mixed_precision.global_policy().variable_dtype}')\n    \n    with STRATEGY.scope():\n        # Set seed for deterministic weights initialization\n        seed_everything(seed=SEED)\n        \n        # Inputs, note the names are equal to the dictionary keys in the dataset\n        image = tf.keras.layers.Input(INPUT_SHAPE, name='image', dtype=tf.uint8)\n        \n        # Normalize Input\n        image_norm = normalize(image)\n\n        # CNN Prediction in range [0,1]\n        x = convnext.ConvNeXtV2Tiny(\n            input_shape=(IMG_HEIGHT, IMG_WIDTH, 3),\n            pretrained='imagenet21k-ft1k',\n            num_classes=0,\n        )(image_norm)\n        \n        # Average Pooling BxHxWxC -> BxC\n        # x = tf.keras.layers.GlobalAveragePooling2D()(x)\n        x = GeMPoolingLayer(p=3.)(x)\n        # Dropout to prevent Overfitting\n        x = tf.keras.layers.Dropout(0.30)(x)\n        # Output value between [0, 1] using Sigmoid function\n        outputs = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n\n        # We will use the famous AdamW optimizer for fast learning with weight decay\n        # optimizer = tfa.optimizers.AdamW(learning_rate=LR_MAX, weight_decay=LR_MAX*WD_RATIO, epsilon=1e-6)\n\n        optimizer = tfa.optimizers.AdaBelief(learning_rate=LR_MAX,\n                                             weight_decay=LR_MAX*WD_RATIO,\n                                             rectify=True,\n                                             total_steps=TRAIN_STEPS_PER_EPOCH * N_EPOCHS,\n                                             warmup_proportion=0.1,\n                                             min_lr=LR_MAX/10)\n        # Loss\n        loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)\n        \n        # Metrics\n        metrics = [\n            pF1(),\n            tfa.metrics.F1Score(num_classes=1, threshold=0.50),\n            tf.keras.metrics.Precision(),\n            tf.keras.metrics.Recall(),\n            tf.keras.metrics.AUC(),\n            tf.keras.metrics.BinaryAccuracy(),\n        ]\n\n        model = tf.keras.models.Model(inputs=image, outputs=outputs)\n        \n        model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n\n        return model","metadata":{"papermill":{"duration":0.118498,"end_time":"2023-02-17T04:05:31.230499","exception":false,"start_time":"2023-02-17T04:05:31.112001","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Pretrained File Path: '/kaggle/input/sartorius-training-dataset/model.h5'\n# tf.keras.backend.clear_session()\n# # enable XLA optmizations\n# tf.config.optimizer.set_jit(True)\n\n# model = get_model()","metadata":{"papermill":{"duration":0.109716,"end_time":"2023-02-17T04:05:31.440822","exception":false,"start_time":"2023-02-17T04:05:31.331106","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Plot model summary\n# model.summary()","metadata":{"papermill":{"duration":0.10961,"end_time":"2023-02-17T04:05:31.651867","exception":false,"start_time":"2023-02-17T04:05:31.542257","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Initilization","metadata":{"papermill":{"duration":0.10158,"end_time":"2023-02-17T04:05:31.854876","exception":false,"start_time":"2023-02-17T04:05:31.753296","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# # Validation metric on initialized model\n# _ = model.evaluate(\n#         get_dataset(df_test, val=True),\n#         verbose=VERBOSE,\n#         steps=VAL_STEPS_PER_EPOCH,\n#     )","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.110466,"end_time":"2023-02-17T04:05:32.066898","exception":false,"start_time":"2023-02-17T04:05:31.956432","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Train Output Baseline\n# val_preds = model.predict(\n#         get_dataset(TFRECORDS_VAL, val=True),\n#         verbose=VERBOSE,\n#         steps=128,\n#     ).squeeze()","metadata":{"papermill":{"duration":0.11355,"end_time":"2023-02-17T04:05:32.282361","exception":false,"start_time":"2023-02-17T04:05:32.168811","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Initialized model train predictions: should not be saturated (all 0/1)\n# display(pd.Series(val_preds).describe().to_frame('Value'))","metadata":{"papermill":{"duration":0.110048,"end_time":"2023-02-17T04:05:32.496927","exception":false,"start_time":"2023-02-17T04:05:32.386879","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(15,8))\n# plt.title(f'Validation Predictions Initialized Model')\n# pd.Series(val_preds).plot(kind='hist')\n# plt.xticks(np.arange(0, 1.1, 0.1))\n# plt.grid()\n# plt.show()","metadata":{"papermill":{"duration":0.110714,"end_time":"2023-02-17T04:05:32.709832","exception":false,"start_time":"2023-02-17T04:05:32.599118","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{"papermill":{"duration":0.101721,"end_time":"2023-02-17T04:05:32.916312","exception":false,"start_time":"2023-02-17T04:05:32.814591","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# # Learning rate scheduler with logaritmic warmup and cosine decay\n# def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n    \n#     if current_step < num_warmup_steps:\n#         return lr_max * 0.10 ** (num_warmup_steps - current_step)\n#     else:\n#         progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n#         return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"papermill":{"duration":0.110787,"end_time":"2023-02-17T04:05:33.128666","exception":false,"start_time":"2023-02-17T04:05:33.017879","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Plot the learning rate scheduler\n# def plot_lr_schedule(lr_schedule, epochs):\n#     fig = plt.figure(figsize=(20, 10))\n#     plt.plot([None] + lr_schedule + [None])\n#     # X Labels\n#     x = np.arange(1, epochs + 1)\n#     x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n#     plt.xlim([1, epochs])\n#     plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n#     # Increase y-limit for better readability\n#     plt.ylim([0, max(lr_schedule) * 1.1])\n    \n#     # Title\n#     schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n#     plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n#     # Plot Learning Rates\n#     for x, val in enumerate(lr_schedule):\n#         if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n#             if x < len(lr_schedule) - 1:\n#                 if lr_schedule[x - 1] < val:\n#                     ha = 'right'\n#                 else:\n#                     ha = 'left'\n#             elif x == 0:\n#                 ha = 'right'\n#             else:\n#                 ha = 'left'\n#             plt.plot(x + 1, val, 'o', color='black');\n#             offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n#             plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n#     plt.xlabel('Epoch', size=16, labelpad=5)\n#     plt.ylabel('Learning Rate', size=16, labelpad=5)\n#     plt.grid()\n#     plt.show()\n\n# # Learning rate for encoder\n# LR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# plot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)","metadata":{"papermill":{"duration":0.111829,"end_time":"2023-02-17T04:05:33.341546","exception":false,"start_time":"2023-02-17T04:05:33.229717","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Learning Rate Callback\n# lr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=0)","metadata":{"papermill":{"duration":0.11098,"end_time":"2023-02-17T04:05:33.553426","exception":false,"start_time":"2023-02-17T04:05:33.442446","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Decay Callback","metadata":{"papermill":{"duration":0.102085,"end_time":"2023-02-17T04:05:33.756625","exception":false,"start_time":"2023-02-17T04:05:33.65454","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# # Tensorflow Learning Rate Scheduler does not update weight decay, need to do it manually in a custom callback\n# class WeightDecayCallback(tf.keras.callbacks.Callback):\n#     def __init__(self, wd_ratio=WD_RATIO):\n#         self.step_counter = 0\n#         self.wd_ratio = wd_ratio\n    \n#     def on_epoch_begin(self, epoch, logs=None):\n#         model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n#         print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"papermill":{"duration":0.116835,"end_time":"2023-02-17T04:05:33.974765","exception":false,"start_time":"2023-02-17T04:05:33.85793","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.103583,"end_time":"2023-02-17T04:05:34.182611","exception":false,"start_time":"2023-02-17T04:05:34.079028","status":"completed"},"tags":[]}},{"cell_type":"code","source":"oof_pred_test = []\noof_targ_test = []\noof_pred_train = []\noof_targ_train = []\noof_history = []\n\nfor fold in np.random.choice(N_SPLITS, 5):\n    SEED = SEED+fold\n    # df_f = train[(train['fold'] != fold) & (train['cancer'] == 0)]\n    # df_t = train[(train['fold'] != fold) & (train['cancer'] == 1)]#.sample(frac=UPSAMPLE, replace=True, random_state=SEED)\n    # df_train = pd.concat([df_f, df_t]).sample(frac=1, random_state=SEED).reset_index(drop=True)\n    df_train = train.copy().sample(frac=1, random_state=SEED).reset_index(drop=True)\n\n    df_f = train[(train['fold'] == fold) & (train['cancer'] == 0)]\n    df_t = train[(train['fold'] == fold) & (train['cancer'] == 1)]#.sample(frac=UPSAMPLE, replace=True, random_state=SEED)\n    df_test = pd.concat([df_f, df_t]).sample(frac=1, random_state=SEED).reset_index(drop=True)\n\n    # Get Train/Validation datasets\n    train_dataset = get_dataset(df_train, val=False, debug=False)\n    val_dataset = get_dataset(df_test, val=True, debug=False)\n    TRAIN_STEPS_PER_EPOCH = len(df_train) // BATCH_SIZE\n    VAL_STEPS_PER_EPOCH = len(df_test) // BATCH_SIZE\n    \n    print('#'*40); print('#### FOLD: ',fold)\n    # enable XLA optmizations    \n    tf.keras.backend.clear_session()\n    tf.config.optimizer.set_jit(True)\n\n    model = get_model()\n\n    print('Training...')\n    history = model.fit(\n        train_dataset,\n        steps_per_epoch = TRAIN_STEPS_PER_EPOCH,\n        validation_data = val_dataset,\n        epochs = N_EPOCHS,\n        verbose = VERBOSE,\n        # callbacks = [\n        #     lr_callback,\n        #     WeightDecayCallback(),\n        # ],\n        class_weight = {\n            0: 1.0,\n            1: 5.0,\n        },\n    )\n    \n    oof_history.append(pd.DataFrame(history.history))\n    display(oof_history[-1])\n    \n    print('Saving best model...')\n    model.save_weights('model_fold-%i.h5'%fold)\n    \n    ds_pred = get_dataset(df_test, val=True, debug=False)\n    pred = model.predict(ds_pred, verbose=0)\n    oof_pred_test.append(pred.ravel())\n    oof_targ_test.append(df_test['cancer'].to_numpy().ravel())\n    pF1_test = pfbeta(oof_targ_test[-1], oof_pred_test[-1])\n    pF1_test_th, thr = pfbeta_thr(oof_targ_test[-1], oof_pred_test[-1], get_thr=True)\n    \n    ds_pred = get_dataset(df_train, val=True, debug=False)\n    pred = model.predict(ds_pred, verbose=0)\n    oof_pred_train.append(pred.ravel())\n    oof_targ_train.append(df_train['cancer'].to_numpy().ravel())\n    pF1_train = pfbeta(oof_targ_train[-1], oof_pred_train[-1])\n    pF1_train_th, _ = pfbeta_thr(oof_targ_train[-1], oof_pred_train[-1], get_thr=True)\n    \n    print('>>>> FOLD %i OOF TEST_pF1 = %.3f, TRAIN_pF1 = %.3f, TEST_pF1_th = %.3f, TRAIN_pF1_th = %.3f, THRESHOLD = %.3f'%(fold,pF1_test,pF1_train,pF1_test_th,pF1_train_th,thr))","metadata":{"papermill":{"duration":22820.77151,"end_time":"2023-02-17T10:25:55.057193","exception":false,"start_time":"2023-02-17T04:05:34.285683","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# F1 By Threshold","metadata":{"papermill":{"duration":0.128913,"end_time":"2023-02-17T10:25:55.869145","exception":false,"start_time":"2023-02-17T10:25:55.740232","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# source: https://www.kaggle.com/code/sohier/probabilistic-f-score\n# Competition Leaderboard Metric\ndef pfbeta(labels, predictions, beta=1):\n    y_true_count = 0\n    ctp = 0\n    cfp = 0\n\n    for idx in range(len(labels)):\n        prediction = min(max(predictions[idx], 0), 1)\n        if (labels[idx]):\n            y_true_count += 1\n            ctp += prediction\n        else:\n            cfp += prediction\n\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0\n\ndef pfbeta_thr(labels, preds, get_thr=False):\n    thrs = tf.range(0, 1, 0.05)\n    best_score = tf.constant(0, dtype=tf.float32)\n    best_thr = tf.constant(0, dtype=tf.float32)\n    \n    for thr in thrs:\n        score = tf.cast(pfbeta(labels, tf.cast(preds>thr, tf.float32)), tf.float32)\n        best_score = tf.cond(score > best_score, lambda: score, lambda: best_score)\n        best_thr = tf.cond(score == best_score, lambda: thr, lambda: best_thr)\n    \n    if get_thr:\n        return best_score, best_thr\n    return best_score","metadata":{"papermill":{"duration":0.144732,"end_time":"2023-02-17T10:25:56.142331","exception":false,"start_time":"2023-02-17T10:25:55.997599","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get true labels and predictions for validation set\ny_true_val = np.concatenate(oof_targ_test, axis=0)\ny_pred_val = np.concatenate(oof_pred_test, axis=0)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.145029,"end_time":"2023-02-17T10:25:56.416549","exception":false,"start_time":"2023-02-17T10:25:56.27152","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot pF1 by threshold plot to find best threshold\npf1_by_threshold = []\nthresholds = np.arange(0, 1.01, 0.01)\nfor t in thresholds:\n    # Compute pF1 for each threshold\n    pf1_by_threshold.append(\n        pfbeta(y_true_val, y_pred_val > t)\n    )\n    \nplt.figure(figsize=(15,8))\nplt.title('F1 By Threshold', size=24)\nplt.plot(pf1_by_threshold, label='F1 Score')\n\n# Best threshold and pF1 score\narg_max = np.argmax(pf1_by_threshold)\nval_max = np.max(pf1_by_threshold)\nthreshold_best = thresholds[arg_max]\nplt.scatter(arg_max, val_max, color='red', label=f'Best Threshold {threshold_best:.2f}, pF1 Score: {val_max:.2f}')\n\n# Plot pF1 by Threshold\nplt.xticks(np.arange(0, 110, 10), [f'{t:.2f}' for t in np.arange(0, 1.1, 0.1)])\nplt.yticks(np.arange(0, 1.1, 0.1))\nplt.xlim(0, 100)\nplt.ylim(0, 1)\nplt.xlabel('Threshold')\nplt.ylabel('pF1 Score')\nplt.legend(fontsize=12)\nplt.grid()\nplt.show()","metadata":{"papermill":{"duration":13.221736,"end_time":"2023-02-17T10:26:09.771503","exception":false,"start_time":"2023-02-17T10:25:56.549767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get true labels and predictions for validation set\ny_true_train = []\ny_pred_train = []\ny_true_train = np.concatenate(oof_targ_train, axis=0)\ny_pred_train = np.concatenate(oof_pred_train, axis=0)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.14241,"end_time":"2023-02-17T10:26:10.048031","exception":false,"start_time":"2023-02-17T10:26:09.905621","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot pF1 by threshold plot to find best threshold\npf1_by_threshold = []\nthresholds = np.arange(0, 1.01, 0.01)\nfor t in thresholds:\n    # Compute pF1 for each threshold\n    pf1_by_threshold.append(\n        pfbeta(y_true_train, y_pred_train > t)\n    )\n    \nplt.figure(figsize=(15,8))\nplt.title('F1 By Threshold', size=24)\nplt.plot(pf1_by_threshold, label='F1 Score')\n\n# Best threshold and pF1 score\narg_max = np.argmax(pf1_by_threshold)\ntrain_max = np.max(pf1_by_threshold)\nthreshold_best = thresholds[arg_max]\nplt.scatter(arg_max, train_max, color='red', label=f'Best Threshold {threshold_best:.2f}, pF1 Score: {train_max:.2f}')\n\n# Plot pF1 by Threshold\nplt.xticks(np.arange(0, 110, 10), [f'{t:.2f}' for t in np.arange(0, 1.1, 0.1)])\nplt.yticks(np.arange(0, 1.1, 0.1))\nplt.xlim(0, 100)\nplt.ylim(0, 1)\nplt.xlabel('Threshold')\nplt.ylabel('pF1 Score')\nplt.legend(fontsize=12)\nplt.grid()\nplt.show()","metadata":{"papermill":{"duration":113.382956,"end_time":"2023-02-17T10:28:03.559765","exception":false,"start_time":"2023-02-17T10:26:10.176809","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History","metadata":{"papermill":{"duration":0.137652,"end_time":"2023-02-17T10:28:03.839748","exception":false,"start_time":"2023-02-17T10:28:03.702096","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax, ylim=None, yscale=None, yticks=None):\n    plt.figure(figsize=(20, 10))\n    \n    values = history.history[metric]\n    N_EPOCHS = len(values)\n    val = 'val' in ''.join(history.history.keys())\n    # Epoch Ticks\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n\n    x_ticks = np.arange(1, N_EPOCHS+1)\n\n    # Validation\n    if val:\n        val_values = history.history[f'val_{metric}']\n        val_argmin = f_best(val_values)\n        plt.plot(x_ticks, val_values, label=f'val')\n\n    # summarize history for accuracy\n    plt.plot(x_ticks, values, label=f'train')\n    argmin = f_best(values)\n    plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label=f'train_best')\n    if val:\n        plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label=f'val_best')\n\n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n\n    if ylim:\n        plt.ylim(ylim)\n\n    if yscale is not None:\n        plt.yscale(yscale)\n        \n    if yticks is not None:\n        plt.yticks(yticks, fontsize=16)\n\n    plt.xlabel('epoch', fontsize=20, labelpad=10)        \n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    \n    plt.legend(prop={'size': 10})\n    plt.grid()\n    plt.show()","metadata":{"papermill":{"duration":0.150305,"end_time":"2023-02-17T10:28:04.121433","exception":false,"start_time":"2023-02-17T10:28:03.971128","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('loss', f_best=np.argmin)","metadata":{"papermill":{"duration":0.506944,"end_time":"2023-02-17T10:28:04.7597","exception":false,"start_time":"2023-02-17T10:28:04.252756","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('pF1', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"papermill":{"duration":0.499842,"end_time":"2023-02-17T10:28:05.391948","exception":false,"start_time":"2023-02-17T10:28:04.892106","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('f1_score', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"papermill":{"duration":0.513923,"end_time":"2023-02-17T10:28:06.047078","exception":false,"start_time":"2023-02-17T10:28:05.533155","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('precision', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"papermill":{"duration":0.527325,"end_time":"2023-02-17T10:28:06.706393","exception":false,"start_time":"2023-02-17T10:28:06.179068","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('recall', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"papermill":{"duration":0.571448,"end_time":"2023-02-17T10:28:07.419883","exception":false,"start_time":"2023-02-17T10:28:06.848435","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('auc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"papermill":{"duration":0.566892,"end_time":"2023-02-17T10:28:08.131749","exception":false,"start_time":"2023-02-17T10:28:07.564857","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('binary_accuracy', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"papermill":{"duration":0.544153,"end_time":"2023-02-17T10:28:08.830414","exception":false,"start_time":"2023-02-17T10:28:08.286261","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":1.525749,"end_time":"2023-02-17T10:28:10.50363","exception":false,"start_time":"2023-02-17T10:28:08.977881","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.141608,"end_time":"2023-02-17T10:28:10.789671","exception":false,"start_time":"2023-02-17T10:28:10.648063","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}