{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112899,"databundleVersionId":13449579,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Grand X-Ray Slam Division A: Transfer learning with ResNet-50\n\nThis notebook shows how to apply [transfer learning](https://en.wikipedia.org/wiki/Transfer_learning) with ResNet-50 to diagnose the X-ray images of the competition.\n\nReferences\n- Competition: [Grand X-Ray Slam: Division A](https://www.kaggle.com/competitions/grand-xray-slam-division-a)\n- The ResNet paper: [Deep Residual Learning for Image Recognition](https://arxiv.org/pdf/1512.03385)\n- [ResNet-50 API documentation](https://www.tensorflow.org/api_docs/python/tf/keras/applications/ResNet50)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm, trange\nimport threading\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import roc_auc_score, RocCurveDisplay\nfrom sklearn.calibration import CalibrationDisplay\n\nimport cv2\nimport os\n\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.layers import Input, Dense, Dropout, Concatenate\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, TerminateOnNaN\nfrom tensorflow.keras.metrics import AUC\nimport tensorflow.keras.applications.resnet50 as resnet50","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-08-30T16:50:33.78626Z","iopub.execute_input":"2025-08-30T16:50:33.78662Z","iopub.status.idle":"2025-08-30T16:50:56.048685Z","shell.execute_reply.started":"2025-08-30T16:50:33.786594Z","shell.execute_reply":"2025-08-30T16:50:56.047609Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_training_history(history):\n    \"\"\"Plot a Keras training history.\"\"\"\n    if len(history['loss']) >= 2:\n        _, axs = plt.subplots(2, 1, figsize=(6, 6))\n        axs[0].plot(history['loss'], ':', label='train_loss')\n        axs[0].plot(history['val_loss'], label='val_loss')\n        axs[0].legend()\n        axs[1].plot(history['AUC'], ':', label='train_auc')\n        axs[1].plot(history['val_AUC'], label='val_auc')\n        axs[1].legend()\n        plt.tight_layout()\n        plt.suptitle('Training history', y=1)\n        plt.show()\n","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-08-30T16:57:59.180653Z","iopub.execute_input":"2025-08-30T16:57:59.181008Z","iopub.status.idle":"2025-08-30T16:57:59.189263Z","shell.execute_reply.started":"2025-08-30T16:57:59.180975Z","shell.execute_reply":"2025-08-30T16:57:59.187582Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_oof(y_train, oof_pred, baseline_true=None, baseline_oof_pred=None, label='', plot=True):\n    \"\"\"Compute oof scores and visualize them\"\"\"\n    n_targets = y_train.shape[1]\n    n_horizontal = (n_targets + 1) // 2 # diagrams per row\n    \n    # Process y_train and oof_pred\n    if type(oof_pred) is not pd.DataFrame:\n        oof_pred = pd.DataFrame(oof_pred, columns=y_train.columns)\n    assert y_train.shape == oof_pred.shape, (y_train.shape, oof_pred.shape)\n    valid_scores = np.ones(n_targets, dtype=bool)\n    scores = np.full(n_targets, 0.5)\n    for i, target in enumerate(y_train.columns):\n        try:\n            scores[i] = roc_auc_score(y_train[target], oof_pred[target])\n        except ValueError:\n            scores[i] = np.nan\n            valid_scores[i] = False\n    macro_mean = np.nanmean(scores[:14])\n\n    # Process baseline_true and baseline_oof_pred\n    if baseline_oof_pred is not None and type(baseline_oof_pred) is not pd.DataFrame:\n        baseline_oof_pred = pd.DataFrame(baseline_oof_pred, columns=y_train.columns)\n        assert baseline_true.shape == baseline_oof_pred.shape, (baseline_true.shape, baseline_oof_pred.shape) \n        baseline_scores = np.array(roc_auc_score(baseline_true, baseline_oof_pred, average=None))\n\n    # Print\n    print(f\"{label}\\t{macro_mean:.3f}\", end='')\n    for i in range(len(scores)):\n        print(f\"\\t{scores[i]:.3f}\", end='')\n    print()\n\n    if plot:\n        # Bar chart\n        plt.figure(figsize=(16, 3))\n        plt.title(f\"Average AUC: {macro_mean:.3f} ({label})\")\n        if baseline_oof_pred is not None:\n            baseline_to_plot = np.where(valid_scores, baseline_scores, np.nan)\n            plt.bar(np.arange(len(scores)) - 0.2, baseline_to_plot - 0.5, bottom=0.5, width=0.4, color='lightgray')\n            bars = plt.bar(np.arange(len(scores)) + 0.2, scores - 0.5, bottom=0.5, width=0.4, color='darkgreen')\n        else:\n            bars = plt.bar(np.arange(len(scores)), scores - 0.5, bottom=0.5, width=0.8, color='lightgray')\n        plt.bar_label(bars, fmt='%.3f')\n        if n_targets > 14:\n            plt.axvline(13.5, color='gray')\n        plt.xticks(np.arange(len(scores)), y_train.columns, rotation=45, ha='right')\n        plt.ylim(0.5, 1)\n        plt.show()\n    \n        # ROC curves\n        _, axs = plt.subplots(2, n_horizontal, figsize=(14, 4))\n        for i, (target, ax) in enumerate(zip(y_train.columns, axs.ravel())):\n            if valid_scores[i]:\n                RocCurveDisplay.from_predictions(y_train[target], oof_pred[target], ax=ax)\n                ax.get_legend().remove()\n                ax.set_title(target)\n                ax.set_xlabel(None) # false positive rate\n                ax.set_ylabel(None) # true positive rate\n                ax.set_aspect('equal')\n            else:\n                ax.set_visible(False)\n        if n_targets < len(axs.ravel()):\n            axs[-1, -1].set_visible(False)\n        plt.tight_layout()\n        plt.suptitle('Receiver operating curves', y=1)\n        plt.show()\n    \n        # Histograms\n        _, axs = plt.subplots(2, n_horizontal, figsize=(14, 3))\n        for i, (target, ax) in enumerate(zip(y_train.columns, axs.ravel())):\n            if valid_scores[i]:\n                ax.hist(oof_pred[target], bins=np.linspace(0, 1, 100), color='brown', density=True)\n                ax.set_title(target)\n                ax.set_xlabel(None) # predicted probability\n                ax.set_ylabel(None) # density\n            else:\n                ax.set_visible(False)\n        if n_targets < len(axs.ravel()):\n            axs[-1, -1].set_visible(False)\n    \n        plt.tight_layout()\n        plt.suptitle('Predicted probability histograms', y=1.01)\n        plt.show()\n\n        # Calibration curves\n        _, axs = plt.subplots(2, n_horizontal, figsize=(14, 4))\n        for i, (target, ax) in enumerate(zip(y_train.columns, axs.ravel())):\n            if valid_scores[i]:\n                CalibrationDisplay.from_predictions(y_train[target], oof_pred[target], \n                                                    ax=ax, color='g',\n                                                    n_bins=10, strategy='quantile')\n                ax.get_legend().remove()\n                ax.set_title(target)\n                ax.set_xlabel(None) # predicted probability\n                ax.set_ylabel(None) # true probability\n                ax.set_aspect('equal')\n            else:\n                ax.set_visible(False)\n        if n_targets < len(axs.ravel()):\n            axs[-1, -1].set_visible(False)\n    \n        plt.tight_layout()\n        plt.suptitle('Calibration display', y=1)\n        plt.show()\n","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-08-30T16:57:59.190542Z","iopub.execute_input":"2025-08-30T16:57:59.190866Z","iopub.status.idle":"2025-08-30T16:57:59.216753Z","shell.execute_reply.started":"2025-08-30T16:57:59.190841Z","shell.execute_reply":"2025-08-30T16:57:59.215843Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Reading the metadata\n\nWe read the metadata into the following dataframes:\n- `train`: image names and features for the training dataset\n- `y_train`: binary targets (ground truth)\n- `test`: image names and features for the test dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/grand-xray-slam-division-a/train1.csv')\ntest = pd.read_csv('/kaggle/input/grand-xray-slam-division-a/sample_submission_1.csv')[['Image_name']]\ntargets = list(train.columns[-14:])\n\nprint(\"Targets:\", targets)\n\ny_train = train[targets].copy()\n\n# Just for fun, we add three more targets:\ny_train['male'] = np.where(train.Sex == 'Male', 1, 0) # must be binary classification\ny_train['female'] = np.where(train.Sex == 'Female', 1, 0) # must be binary classification\ny_train['frontal'] = (train.ViewCategory == 'Frontal').astype(float)\n\ndef engineer_features(df):\n    df['Patient_ID'] = df['Image_name'].str.slice(0, 8).astype(int)\n    df['Study'] = df['Image_name'].str.slice(9, 12).astype(int)\n    df['Patient_Study'] = df['Image_name'].str.slice(0, 12).astype(int)\n    df['institute'] = df['Patient_ID'] // 10000000\n    return df\n\nengineer_features(train)\nengineer_features(test)\ndisplay(test.tail(4))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T16:51:40.205317Z","iopub.execute_input":"2025-08-30T16:51:40.205693Z","iopub.status.idle":"2025-08-30T16:51:40.864396Z","shell.execute_reply.started":"2025-08-30T16:51:40.205664Z","shell.execute_reply":"2025-08-30T16:51:40.863674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Test the notebook with a subset of the data\n# train = train.iloc[:1000]\n# y_train = y_train.iloc[:1000]\n# test = test.iloc[:60]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T16:51:43.930382Z","iopub.execute_input":"2025-08-30T16:51:43.931555Z","iopub.status.idle":"2025-08-30T16:51:43.936823Z","shell.execute_reply.started":"2025-08-30T16:51:43.931496Z","shell.execute_reply":"2025-08-30T16:51:43.935667Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing\n\nData preprocessing is tha part of the notebook which takes the most time. We read 200 GByte of images from an NFS server, resize them, and feed them through a 50-layer neural network. Fortunately we don't need to train the 50-layer ResNet ourselves: The model has been pretrained on other images. \n\nThis is the essence of transfer learning: Somebody else has trained the model with some other images, and we transfer the network's learned weights to our own application (diagnosing X-ray images). ResNet-50 computes a 2048-feature vector (an embedding) for every image, and we only train a small classifier (the \"head\") on these features. \n\nResNet-50 is an ordinary Keras model. You can print its summary with `resnet50_model.summary()`.\n\nWe save the preprocessing results (i.e., the embeddings of all images) to disk so that in the next version of the notebook we can skip preprocessing. If you copy the notebook, you can save some time by simply reading the two files from the output of this public notebook.","metadata":{}},{"cell_type":"code","source":"# %%time\nn_threads = 10 # set to 1 for debugging, 10 for production\nwidth = 224 # must be divisible by 32\nheight = 224 # must be divisible by 32\nresnet_embedding_dim = 2048\n\ndef load_and_resize_images(n_threads, j, paths, images, width, height):\n    \"\"\"Load and resize a subset of the images.\n\n    This function is to be executed by several threads in parallel.\n\n    The images are resized to width x height pixels. If the original image\n    isn't square, the pixels are stretched.\n    \"\"\"\n    for im_idx in range(j, len(paths), n_threads):\n        img = cv2.imread(paths[im_idx], cv2.IMREAD_GRAYSCALE)\n        assert img is not None\n        img = cv2.resize(img, (width, height)) # resize expects (width, height)\n        # if n_threads == 1 and im_idx < 10:\n        #     print(img.shape, img.dtype) # array has shape (height, width) and dtype uint8\n        #     plt.imshow(img, vmin=0, vmax=255)\n        #     plt.show()\n        images[im_idx] = img\n\n\ndef load_and_resize_dataset(directory, df, width, height):\n    \"\"\"Load and resize all images of a dataset.\n    \n    The images are read by several threads in parallel: Reading the images is i/o-bound, and\n    multithreading gives good speedup. Unfortunately, multithreading is incompatible with tqdm.\n    The process can take an hour, and you won't see a progress bar.\n    \"\"\"\n    assert width % 32 == 0, 'must be divisible by 32' \n    assert height % 32 == 0, 'must be divisible by 32'\n    image_dir = f'/kaggle/input/grand-xray-slam-division-a/{directory}/'\n    paths = [f'/kaggle/input/grand-xray-slam-division-a/{directory}/{p}' for p in df.Image_name]\n    print(f\"{directory}: Processing {len(df)} images\")\n\n    # Create an array of resized images of shape (n_images, height, width) and dtype uint8\n    images = np.zeros((len(df), height, width), dtype=np.uint8)\n\n    if n_threads > 1:\n        threads = [threading.Thread(target=load_and_resize_images, args=(n_threads, j, paths, images, width, height)) for j in range(n_threads)]\n        for thread in threads: thread.start()\n        for thread in threads: thread.join()\n    else: # debug in the main thread\n        load_and_resize_images(1, 0, paths, images, width, height)\n\n    assert (images.sum(axis=2).sum(axis=1) != 0).all()\n    return images\n\n    \ndef embed_with_ResNet(images):\n    \"\"\"Read the uint8 array of images and compute embeddings\"\"\"\n\n    embeddings = np.zeros((len(images), resnet_embedding_dim), dtype=np.float32)\n    batch_size = 10240\n    for batch_start in trange(0, len(images), batch_size):\n        \n        # Convert grayscale to rgb and center\n        # X = img_train # shape (n_samples, HEIGHT, WIDTH) of uint8\n        Xb = images[batch_start:batch_start+batch_size]\n        XX = np.repeat(Xb, 3).reshape(Xb.shape + (3, )) # shape (n_samples, HEIGHT, WIDTH, 3)\n        XX = resnet50.preprocess_input(XX)\n        \n        # Compute embeddings\n        embeddings[batch_start:batch_start+batch_size] = resnet50_model.predict(XX, batch_size=256, verbose=1) # shape (n_samples, 2048)\n    return embeddings\n\n\ndef load_resize_embed(directory, df, width, height):\n    images = load_and_resize_dataset(directory, df, width, height)\n    embeddings = embed_with_ResNet(images)\n    print(f\"embeddings_{directory[:-1]}.shape = {embeddings.shape}\")\n    \n    embeddings.tofile(f\"embeddings_{directory[:-1]}.binary\")\n    loading = f\"np.memmap(f'embeddings_{directory[:-1]}.binary', mode='r', dtype=np.float32, shape={embeddings.shape})\"\n    embeddings_2 = eval(loading)\n    assert (embeddings == embeddings_2).all()\n    print(f\"Next time, you can load the embeddings with\\n\\n    embeddings_{directory[:-1]} = {loading}\\n\")\n\n    return embeddings\n\n\n# Load the ResNet50 model\nresnet50_model = resnet50.ResNet50(include_top=False,\n                               weights='imagenet',\n                               pooling='avg',\n                               input_shape=(height, width, 3))\n\n# Print the model summary\n# resnet50_model.summary()\n\n# Do the job\nembeddings_train = load_resize_embed('train1', train, width, height)\nembeddings_test = load_resize_embed('test1', test, width, height)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T16:51:44.572457Z","iopub.execute_input":"2025-08-30T16:51:44.573476Z","iopub.status.idle":"2025-08-30T16:54:25.358061Z","shell.execute_reply.started":"2025-08-30T16:51:44.573432Z","shell.execute_reply":"2025-08-30T16:54:25.356848Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# The Keras PyDataset\n\nThis class reads the embeddings and returns them in batches together with the metadata.","metadata":{}},{"cell_type":"code","source":"# Custom data generator for images\n\nclass XRayGeneratorForResNet(Sequence):\n    \"\"\"Sequence of image batches.\n\n    The images are read from disk with fixed width and height. It is possible that\n    pixels are no longer square after the transformation.\n\n    X and y have the same index; this index corresponds to the positions of the images in the array.\n    \"\"\"\n    def __init__(self, X, y=None, embeddings=None, shuffle=False, batch_size=32):\n        super().__init__(workers=1)\n        assert embeddings is not None\n        if y is not None:\n            assert len(X) == len(y)\n        self.X = X # dataframe with columns 'Image_name' and 'institute'\n        self.y = y # dataframe with one column per target\n        self.embeddings = embeddings\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n\n        # Shuffle to avoid all images in a batch being of the same patient\n        self.permutation = np.arange(len(self.X))\n        if self.shuffle:\n            self.rng = np.random.default_rng(1)\n            self.rng.shuffle(self.permutation)\n\n    def __len__(self):\n        \"\"\"Number of batches in one epoch, including a smaller batch at the end.\"\"\"\n        return (len(self.X) + self.batch_size - 1) // self.batch_size\n\n    def __getitem__(self, idx):\n        \"\"\"Return a batch of images (and optionally, labels).\n\n        Returns\n        images: float32 array of shape (batch_size, resnet_embedding_dim)\n        metadata: float32 array of shape (batch_size, 1)\n        labels: float32 array of shape (batch_size, n_targets)\n        \"\"\"\n        start_idx = idx * self.batch_size\n        end_idx = min(start_idx + self.batch_size, len(self.X))\n        batch_indexes = self.permutation[start_idx:end_idx]\n        resnet_embeddings = self.embeddings[self.X.index[batch_indexes]]\n        metadata = self.X[['institute']].values[batch_indexes].astype(np.float32)\n        features = {'resnet_embeddings': resnet_embeddings, 'metadata': metadata}\n\n        if self.y is not None:\n            labels = self.y.values[batch_indexes].astype(np.float32)\n            return features, labels\n        return features\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T16:54:25.360377Z","iopub.execute_input":"2025-08-30T16:54:25.360748Z","iopub.status.idle":"2025-08-30T16:54:25.371754Z","shell.execute_reply.started":"2025-08-30T16:54:25.360723Z","shell.execute_reply":"2025-08-30T16:54:25.370494Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# The model\n\nWe use a fully connected model with two hidden layers as our classifier. It takes the embeddings and some metadata as input, and it predicts probabilities for all classes.","metadata":{}},{"cell_type":"code","source":"def create_head_model(num_classes=y_train.shape[1]):\n    resnet_embeddings = Input((resnet_embedding_dim, ), name='resnet_embeddings')\n    metadata =  Input((1, ), name='metadata')\n    x = Concatenate()([resnet_embeddings, metadata])\n    x = Dense(512, activation='relu')(x)\n    x = Dropout(0.3)(x)\n    x = Dense(128, activation='relu')(x)\n    x = Dense(num_classes, activation='sigmoid')(x)\n    model = Model(inputs={'resnet_embeddings': resnet_embeddings,\n                          'metadata': metadata},\n                  outputs=x,\n                  name='Resnet50-Head')\n    return model    \n\ncreate_head_model().summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T16:54:25.372632Z","iopub.execute_input":"2025-08-30T16:54:25.372973Z","iopub.status.idle":"2025-08-30T16:54:25.465629Z","shell.execute_reply.started":"2025-08-30T16:54:25.372944Z","shell.execute_reply":"2025-08-30T16:54:25.464839Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model fitting and inference\n\nWe run a five-fold cross-validation of our model. Because the test patients are disjoint from the training patients, we use a GroupKFold.\n\nTo do:\n1. Study the diagrams and find ideas to improve the model. Which target is the easiest to predict and which is the hardest?\n2. Compare the cross-validation score to the leaderboard score. Do the scores match?","metadata":{}},{"cell_type":"code","source":"# %%time\n\n# Parameters\nepochs, n_folds, compute_test_predictions = 50, 5, True # for submission\n# epochs, n_folds, compute_test_predictions = 50, 1, False # for validation without submission\nbatch_size = 64\nverbose = 1 if os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive' else 2\n\nif compute_test_predictions:\n    test_generator = XRayGeneratorForResNet(test, None, embeddings_test, batch_size=batch_size, shuffle=False)\n    test_pred_list = []\n\ngkf = GroupKFold()\nfor fold, (idx_tr, idx_va) in enumerate(gkf.split(train, groups=train.Patient_ID)):\n    if fold == n_folds: break\n\n    # Split into train and test and construct the generators\n    X_tr = train.iloc[idx_tr]\n    X_va = train.iloc[idx_va]\n    y_tr = y_train.iloc[idx_tr]\n    y_va = y_train.iloc[idx_va]\n    train_generator = XRayGeneratorForResNet(X_tr, y_tr, embeddings_train, batch_size=batch_size, shuffle=True)\n    val_generator = XRayGeneratorForResNet(X_va, y_va, embeddings_train, batch_size=batch_size, shuffle=False)\n    \n    # Construct and compile the model\n    model = create_head_model()\n    label = f\"{model.name}\"\n    print('Fitting', label)\n    label_weights = (y_train.var(axis=0) != 0).astype(float) # exclude constant targets from metric\n    label_weights[14:] = 0 # exclude just-for-fun targets from metric\n    model.compile(\n        # optimizer=Adam(learning_rate=0.001),\n        optimizer=Adam(learning_rate=0.001),\n        loss='binary_crossentropy',\n        metrics=[AUC(multi_label=True, name='AUC', label_weights=label_weights)]\n    )\n    \n    # Train and plot the training history\n    history = model.fit(\n        train_generator,\n        validation_data=val_generator,\n        epochs=epochs,\n        callbacks=[EarlyStopping(patience=3),\n                   ReduceLROnPlateau(factor=0.5, patience=0, verbose=1, min_lr=0.000126),\n                   TerminateOnNaN()],\n        verbose=verbose\n    )\n    history = history.history\n    plot_training_history(history)\n\n    def do_inference(generator, X, y_true=None):\n        \"\"\"Compute predictions grouping by study, and evaluate them.\"\"\"\n        y_pred = model.predict(generator, verbose=verbose)\n        if y_true is not None: evaluate_oof(y_true, y_pred, label=label, plot=False)\n        \n        y_pred_frontal = y_pred[:,-1]\n        y_pred = pd.DataFrame(y_pred, index=X.index)\n        y_pred = y_pred.groupby(X['Patient_Study'].values).transform(lambda x: x.values.mean()).values\n        y_pred[:,-1] = y_pred_frontal\n        if y_true is not None: evaluate_oof(y_true, y_pred, label=label+' grouped')\n        return y_pred\n\n    # Validate and visualize validation score\n    print(\"\\nValidation\")\n    do_inference(val_generator, X_va, y_va)\n    print()\n\n    # Compute test_predictions\n    if compute_test_predictions:\n        test_pred_list.append(do_inference(test_generator, test))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T16:57:59.218348Z","iopub.execute_input":"2025-08-30T16:57:59.218884Z","iopub.status.idle":"2025-08-30T16:59:21.467032Z","shell.execute_reply.started":"2025-08-30T16:57:59.218855Z","shell.execute_reply":"2025-08-30T16:59:21.46594Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission\n\nAs we have trained five models for cross-validation, we can now compute the submission as an ensemble of the predictions of five models.","metadata":{}},{"cell_type":"code","source":"if len(test_pred_list) > 0:\n    test_pred = np.stack(test_pred_list).mean(axis=0)\n    submission = pd.DataFrame(test_pred[:,:14],\n                              columns=targets,\n                              index=test.Image_name)\n    display(submission)\n    submission.to_csv('submission.csv')\n    !head submission.csv\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-30T16:55:47.493101Z","iopub.execute_input":"2025-08-30T16:55:47.493456Z","iopub.status.idle":"2025-08-30T16:55:47.714896Z","shell.execute_reply.started":"2025-08-30T16:55:47.49343Z","shell.execute_reply":"2025-08-30T16:55:47.713487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}