{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112899,"databundleVersionId":13449579,"sourceType":"competition"},{"sourceId":13309739,"sourceType":"datasetVersion","datasetId":8436923}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install iterative-stratification","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T08:58:39.031564Z","iopub.execute_input":"2025-10-09T08:58:39.031721Z","iopub.status.idle":"2025-10-09T08:58:43.39576Z","shell.execute_reply.started":"2025-10-09T08:58:39.031707Z","shell.execute_reply":"2025-10-09T08:58:43.394911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nUse EVA-X series as your backbone. You could get \nEVA-X representations simply with timm. Try them \nwith your own X-ray tasks. \nEnjoy!\n\nReference:\n    https://github.com/baaivision/EVA\n    https://github.com/huggingface/pytorch-image-models\nThanks for their work!\n    \nby Jingfeng Yao \nfrom HUST-VL\n\"\"\"\n\nimport torch\nimport torch.nn as nn\nfrom timm.layers import resample_abs_pos_embed, resample_patch_embed\nfrom timm.models.eva import Eva\n\n\ndef checkpoint_filter_fn(\n        state_dict,\n        model,\n        interpolation='bicubic',\n        antialias=True,\n):\n    \"\"\" convert patch embedding weight from manual patchify + linear proj to conv\"\"\"\n    out_dict = {}\n    state_dict = state_dict.get('model_ema', state_dict)\n    state_dict = state_dict.get('model', state_dict)\n    state_dict = state_dict.get('module', state_dict)\n    state_dict = state_dict.get('state_dict', state_dict)\n    # prefix for loading OpenCLIP compatible weights\n    if 'visual.trunk.pos_embed' in state_dict:\n        prefix = 'visual.trunk.'\n    elif 'visual.pos_embed' in state_dict:\n        prefix = 'visual.'\n    else:\n        prefix = ''\n    mim_weights = prefix + 'mask_token' in state_dict\n    no_qkv = prefix + 'blocks.0.attn.q_proj.weight' in state_dict\n\n    len_prefix = len(prefix)\n    for k, v in state_dict.items():\n        if prefix:\n            if k.startswith(prefix):\n                k = k[len_prefix:]\n            else:\n                continue\n\n        if 'rope' in k:\n            # fixed embedding no need to load buffer from checkpoint\n            continue\n\n        if 'patch_embed.proj.weight' in k:\n            _, _, H, W = model.patch_embed.proj.weight.shape\n            if v.shape[-1] != W or v.shape[-2] != H:\n                v = resample_patch_embed(\n                    v,\n                    (H, W),\n                    interpolation=interpolation,\n                    antialias=antialias,\n                    verbose=True,\n                )\n        elif k == 'pos_embed' and v.shape[1] != model.pos_embed.shape[1]:\n            # To resize pos embedding when using model at different size from pretrained weights\n            num_prefix_tokens = 0 if getattr(model, 'no_embed_class', False) else getattr(model, 'num_prefix_tokens', 1)\n            v = resample_abs_pos_embed(\n                v,\n                new_size=model.patch_embed.grid_size,\n                num_prefix_tokens=num_prefix_tokens,\n                interpolation=interpolation,\n                antialias=antialias,\n                verbose=True,\n            )\n\n        k = k.replace('mlp.ffn_ln', 'mlp.norm')\n        k = k.replace('attn.inner_attn_ln', 'attn.norm')\n        k = k.replace('mlp.w12', 'mlp.fc1')\n        k = k.replace('mlp.w1', 'mlp.fc1_g')\n        k = k.replace('mlp.w2', 'mlp.fc1_x')\n        k = k.replace('mlp.w3', 'mlp.fc2')\n        if no_qkv:\n            k = k.replace('q_bias', 'q_proj.bias')\n            k = k.replace('v_bias', 'v_proj.bias')\n\n        if mim_weights and k in ('mask_token', 'lm_head.weight', 'lm_head.bias', 'norm.weight', 'norm.bias'):\n            if k == 'norm.weight' or k == 'norm.bias':\n                # try moving norm -> fc norm on fine-tune, probably a better starting point than new init\n                k = k.replace('norm', 'fc_norm')\n            else:\n                # skip pretrain mask token & head weights\n                continue\n\n        out_dict[k] = v\n\n    return out_dict\n\nclass EVA_X(Eva):\n    def __init__(self, **kwargs):\n        super(EVA_X, self).__init__(**kwargs)\n\n        self.head4 = nn.Linear(self.head.in_features, 4)\n    def forward_features(self, x):\n        x = self.patch_embed(x)\n        x, rot_pos_embed = self._pos_embed(x)\n        for blk in self.blocks:\n            x = blk(x, rope=rot_pos_embed)\n        x = self.norm(x)\n        return x\n\n    def forward_head(self, x, pre_logits: bool = False):\n        if self.global_pool:\n            x = x[:, self.num_prefix_tokens:].mean(dim=1) if self.global_pool == 'avg' else x[:, 0]\n        x = self.fc_norm(x) # LayerNorm\n        x = self.head_drop(x) # Dropout\n        return x if pre_logits else self.head(x) # Linear\n\n    def forward_head4(self, x):\n        if self.global_pool:\n            x = x[:, self.num_prefix_tokens:].mean(dim=1) if self.global_pool == 'avg' else x[:, 0]\n        x = self.fc_norm(x) # LayerNorm\n        x = self.head_drop(x) # Dropout\n        return self.head4(x) # Linear\n\n    def contrastive_loss(feats, labels):\n        return\n    \n    def forward(self, x, out4 = False):\n        feats = self.forward_features(x)\n        x14 = self.forward_head(feats)\n        if out4:\n            x4 = self.forward_head4(feats)\n            return x14, x4\n        else:\n            return x14\n\ndef eva_x_tiny_patch16(pretrained=False):\n    model = EVA_X(\n        img_size=224,\n        patch_size=16,\n        embed_dim=192,\n        depth=12,\n        num_heads=3,\n        mlp_ratio=4 * 2 / 3,\n        swiglu_mlp=True,\n        use_rot_pos_emb=True,\n        ref_feat_shape=(14, 14),  # 224/16\n    )\n    eva_ckpt = checkpoint_filter_fn(torch.load(pretrained, map_location='cpu'), \n                        model)\n    msg = model.load_state_dict(eva_ckpt, strict=False)\n    print(msg)\n    return model\n\ndef eva_x_small_patch16(pretrained=False):\n    model = EVA_X(\n        img_size=224, \n        patch_size=16,\n        embed_dim=384,\n        depth=12,\n        num_heads=6,\n        mlp_ratio=4 * 2 / 3,\n        swiglu_mlp=True,\n        use_rot_pos_emb=True,\n        ref_feat_shape=(14, 14),   # 224/16\n        # num_classes = 14,\n        # drop_path_rate=0.2\n    )\n    eva_ckpt = checkpoint_filter_fn(torch.load(pretrained, map_location='cpu', weights_only=False), \n                        model)\n    msg = model.load_state_dict(eva_ckpt, strict=False)\n    print(msg)\n    return model\n\ndef eva_x_base_patch16(pretrained=False):\n    model = EVA_X(\n        img_size=224,\n        patch_size=16,\n        embed_dim=768,\n        depth=12,\n        num_heads=12,\n        qkv_fused=False,\n        mlp_ratio=4 * 2 / 3,\n        swiglu_mlp=True,\n        scale_mlp=True,\n        use_rot_pos_emb=True,\n        ref_feat_shape=(14, 14),  # 224/16\n        num_classes = 14,\n    )\n    eva_ckpt = checkpoint_filter_fn(torch.load(pretrained, map_location='cpu',weights_only=False), \n                        model)\n    msg = model.load_state_dict(eva_ckpt, strict=False)\n    print(msg)\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T08:58:43.396779Z","iopub.execute_input":"2025-10-09T08:58:43.397084Z","iopub.status.idle":"2025-10-09T08:58:53.729833Z","shell.execute_reply.started":"2025-10-09T08:58:43.397051Z","shell.execute_reply":"2025-10-09T08:58:53.729243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import copy\nimport os\nimport warnings\nimport albumentations as A\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom albumentations.pytorch import ToTensorV2\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nfrom PIL import Image\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import KFold, StratifiedGroupKFold\nfrom timm.data import Mixup\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, LambdaLR\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nfrom tqdm import tqdm\nimport optuna\nfrom functools import partial\n# from evax.eva_x import eva_x_base_patch16, eva_x_small_patch16\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\n\ndef seed_everything_torch(seed=42):\n    import os\n    import random\n    import torch\n    import numpy as np\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything_torch(42)\nwarnings.filterwarnings(\"ignore\")\n\n# --- 1. Configuration ---\nclass Config:\n    DEBUG = False\n    BASE_PATH = \"/kaggle/input/grand-xray-slam-division-a/\"\n    TRAIN_IMG_PATH = os.path.join(BASE_PATH, \"train1\") #train_npy/train_npy\n    TEST_IMG_PATH = os.path.join(BASE_PATH, \"test1\") #test_npy/test_npy\n    TRAIN_CSV = os.path.join(BASE_PATH, \"train1.csv\")\n    SAMPLE_SUB_CSV = os.path.join(BASE_PATH, \"sample_submission_1.csv\")\n    \n    IMAGE_COLUMN_NAME = 'Image_name'\n    VIEW_POSITION_COLUMN = 'ViewPosition'\n    MODEL_NAME = 'convnext_tiny.fb_in22k'\n    IMG_SIZE = 224 \n    BATCH_SIZE = 64\n    EPOCHS_INITIAL = 9\n    EPOCHS_PSEUDO = 3\n    LEARNING_RATE = 1e-4\n    NUM_WORKERS = 2\n    DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    DROP_RATE = 0.10\n    DROP_PATH_RATE = 0.10\n    PSEUDO_CONFIDENCE_HIGH = 0.97\n    PSEUDO_CONFIDENCE_LOW = 0.03\n    DEBUG_PSEUDO_SAMPLE_SIZE = 500\n    DEBUG_FINAL_INFERENCE_SIZE = 50\n    \n    TARGET_LABELS = [\n        'Atelectasis', 'Cardiomegaly', 'Consolidation', 'Edema', \n        'Enlarged Cardiomediastinum', 'Fracture', 'Lung Lesion', \n        'Lung Opacity', 'No Finding', 'Pleural Effusion', 'Pleural Other', \n        'Pneumonia', 'Pneumothorax', 'Support Devices'\n    ]\n    NUM_CLASSES = len(TARGET_LABELS)\n    VIEW_POSITION_LABELS = ['AP', 'PA', 'Lateral', 'LL']\n    NUM_VIEW_CLASSES = len(VIEW_POSITION_LABELS)\n    LOSS_HEAD_WEIGHTS = (0.7, 0.3)\n    NUM_FOLDS = 5\n\n    \nprint(f\"Using device: {Config.DEVICE}\")\nprint(f\"Using model: {Config.MODEL_NAME} with image size {Config.IMG_SIZE}\")\n\n# --- 2. Loss Functions ---\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2.0, reduction='mean'):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n    def forward(self, inputs, targets):\n        bce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n        pt = torch.exp(-bce_loss)\n        focal_loss = self.alpha * (1 - pt)**self.gamma * bce_loss\n        return focal_loss.mean()\n\nclass AsymmetricLossOptimized(nn.Module):\n    def __init__(self, gamma_neg=4, gamma_pos=1, clip=0.05, eps=1e-8, disable_torch_grad_focal_loss=False):\n        super(AsymmetricLossOptimized, self).__init__()\n        self.gamma_neg = gamma_neg\n        self.gamma_pos = gamma_pos\n        self.clip = clip\n        self.disable_torch_grad_focal_loss = disable_torch_grad_focal_loss\n        self.eps = eps\n        self.targets = self.anti_targets = self.xs_pos = self.xs_neg = self.asymmetric_w = self.loss = None\n    def forward(self, x, y):\n        self.targets = y\n        self.anti_targets = 1 - y\n        self.xs_pos = torch.sigmoid(x)\n        self.xs_neg = 1.0 - self.xs_pos\n        if self.clip is not None and self.clip > 0:\n            self.xs_neg.add_(self.clip).clamp_(max=1)\n        self.loss = self.targets * torch.log(self.xs_pos.clamp(min=self.eps))\n        self.loss.add_(self.anti_targets * torch.log(self.xs_neg.clamp(min=self.eps)))\n        if self.gamma_neg > 0 or self.gamma_pos > 0:\n            if self.disable_torch_grad_focal_loss:\n                torch.set_grad_enabled(False)\n            self.xs_pos = self.xs_pos * self.targets\n            self.xs_neg = self.xs_neg * self.anti_targets\n            self.asymmetric_w = torch.pow(1 - self.xs_pos - self.xs_neg,\n                                          self.gamma_pos * self.targets + self.gamma_neg * self.anti_targets)\n            if self.disable_torch_grad_focal_loss:\n                torch.set_grad_enabled(True)\n            self.loss *= self.asymmetric_w\n        return -self.loss.sum()\n    \nclass HybridLoss(nn.Module):\n    def __init__(self, bce_weights=None, alpha=0.3, beta=0.7, gamma_neg=1.2, gamma_pos=2.5, clip=0.05, eps=1e-8):\n        \"\"\"\n        Hybrid loss combining weighted BCE and Asymmetric Loss Optimized (ASL).\n        \n        Args:\n            bce_weights (torch.Tensor, optional): Weights for each class in BCE loss. Shape: (num_classes,).\n            alpha (float): Weight for BCE loss in the hybrid combination.\n            beta (float): Weight for ASL loss in the hybrid combination.\n            gamma_neg (float): Gamma parameter for negative samples in ASL.\n            gamma_pos (float): Gamma parameter for positive samples in ASL.\n            clip (float): Clipping margin for ASL to stabilize negative samples.\n            eps (float): Small value to prevent log(0) in ASL.\n        \"\"\"\n        super(HybridLoss, self).__init__()\n        self.bce_weights = bce_weights if bce_weights is not None else torch.ones(Config.NUM_CLASSES).to(Config.DEVICE)\n        self.alpha = alpha\n        self.beta = beta\n        self.bce = nn.BCEWithLogitsLoss(weight=self.bce_weights, reduction='mean')\n        self.asl = AsymmetricLossOptimized(\n            gamma_neg=gamma_neg,\n            gamma_pos=gamma_pos,\n            clip=clip,\n            eps=eps,\n            disable_torch_grad_focal_loss=False\n        )\n\n    def forward(self, inputs, targets):\n        \"\"\"\n        Compute the hybrid loss.\n        \n        Args:\n            inputs (torch.Tensor): Model predictions (logits), shape: (batch_size, num_classes).\n            targets (torch.Tensor): Ground truth labels, shape: (batch_size, num_classes).\n        \n        Returns:\n            torch.Tensor: Combined loss value.\n        \"\"\"\n        bce_loss = self.bce(inputs, targets)\n        asl_loss = self.asl(inputs, targets)\n        return self.alpha * bce_loss + self.beta * asl_loss\n        \ndef exclusivity_regularizer(p, y, idx_nf, cond_on_ynf=True):\n    mask = torch.ones(p.size(1), dtype=torch.bool, device=p.device)\n    mask[idx_nf] = False\n    p_any = 1.0 - torch.prod(1.0 - p[:, mask], dim=1)\n    reg = p[:, idx_nf] * p_any\n    if cond_on_ynf:\n        reg = reg * y[:, idx_nf]\n    return reg.mean()\n\ndef multilabel_mixup(x, y, alpha=0.2):\n    lam = np.random.beta(alpha, alpha)\n    batch_size = x.size(0)\n    index = torch.randperm(batch_size).to(x.device)\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    mixed_y = lam * y + (1 - lam) * y[index, :]\n    return mixed_x, mixed_y\n\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\ndef multilabel_cutmix(x, y, alpha=1.0):\n    lam = np.random.beta(alpha, alpha)\n    batch_size = x.size(0)\n    index = torch.randperm(batch_size).to(x.device)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(x.size(), lam)\n    x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2]\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.size(-1) * x.size(-2)))\n    y_mix = lam * y + (1 - lam) * y[index, :]\n    return x, y_mix\n\ndef mixup_cutmix(x, y, mixup_alpha=0.2, cutmix_alpha=1.0, prob=1.0, switch_prob=0.5):\n    if np.random.rand() < prob:\n        if np.random.rand() < switch_prob:\n            return multilabel_mixup(x, y, alpha=mixup_alpha)\n        else:\n            return multilabel_cutmix(x, y, alpha=cutmix_alpha)\n    return x, y\n\ndef get_scheduler(optimizer, warmup_epochs, total_epochs, base_lr, min_lr):\n    def warmup_fn(epoch):\n        if epoch < warmup_epochs:\n            return (epoch + 1) / warmup_epochs\n        return 1.0\n    warmup_scheduler = LambdaLR(optimizer, lr_lambda=warmup_fn)\n    cosine_scheduler = CosineAnnealingLR(\n        optimizer,\n        T_max=(total_epochs - warmup_epochs),\n        eta_min=min_lr\n    )\n    return warmup_scheduler, cosine_scheduler\n\n# --- 3. EMA Helper ---\nclass ModelEMA:\n    def __init__(self, model, decay=0.999):\n        self.ema = copy.deepcopy(model).eval()\n        self.decay = decay\n        for p in self.ema.parameters():\n            p.requires_grad_(False)\n    def update(self, model):\n        with torch.no_grad():\n            msd = model.state_dict()\n            for k, v in self.ema.state_dict().items():\n                if v.dtype.is_floating_point:\n                    v.copy_(v * self.decay + msd[k].detach() * (1. - self.decay))\n\n# --- 4. Load Data ---\nprint(\"Loading data...\")\ntrain_df = pd.read_csv(Config.TRAIN_CSV)\nsample_submission_df = pd.read_csv(Config.SAMPLE_SUB_CSV)\n\ndef construct_image_path(base_path, image_name):\n    npy_path = os.path.join(base_path, image_name.replace('.jpg', '.npy'))\n    jpg_path = os.path.join(base_path, image_name)\n    if os.path.exists(npy_path):\n        return npy_path\n    elif os.path.exists(jpg_path):\n        return jpg_path\n    else:\n        raise FileNotFoundError(f\"No file found for {image_name} at {npy_path} or {jpg_path}\")\n\ntrain_df['ImagePath'] = train_df[Config.IMAGE_COLUMN_NAME].apply(\n    lambda x: construct_image_path(Config.TRAIN_IMG_PATH, x)\n)\nsample_submission_df['ImagePath'] = sample_submission_df[Config.IMAGE_COLUMN_NAME].apply(\n    lambda x: construct_image_path(Config.TEST_IMG_PATH, x)\n)\n\nif Config.DEBUG:\n    train_df = train_df.sample(frac=0.01, random_state=42).reset_index(drop=True)\n\nmskf = MultilabelStratifiedKFold(n_splits=Config.NUM_FOLDS, shuffle=True, random_state=42)\nfold_indices = list(mskf.split(train_df, train_df[Config.TARGET_LABELS].values))\n\n# --- 5. Dataset ---\nclass ChestXRayDataset(Dataset):\n    def __init__(self, df, transform=None, is_test=False):\n        self.df = df\n        self.transform = transform\n        self.is_test = is_test\n        self.view_map = {'AP': 0, 'PA': 1, 'Lateral': 2, 'LL': 3}\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_path = self.df.iloc[idx]['ImagePath']\n        # Determine file type based on extension\n        if img_path.endswith('.npy'):\n            image = np.load(img_path).astype(np.uint8)\n            if image.ndim == 2:\n                image = np.stack([image, image, image], axis=-1)\n        elif img_path.endswith('.jpg') or img_path.endswith('.jpeg'):\n            image = cv2.imread(img_path)\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)  # Convert BGR to RGB\n            if image is None:\n                raise FileNotFoundError(f\"Could not load image at {img_path}\")\n        else:\n            raise ValueError(f\"Unsupported file format for {img_path}\")\n\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n\n        if self.is_test:\n            return image\n        else:\n            labels = self.df.iloc[idx][Config.TARGET_LABELS].values.astype(np.float32)\n            view_position = self.df.iloc[idx][Config.VIEW_POSITION_COLUMN]\n            view_idx = self.view_map[view_position]\n            view_one_hot = np.zeros(Config.NUM_VIEW_CLASSES, dtype=np.float32)\n            view_one_hot[view_idx] = 1.0\n            return image, torch.tensor(labels, dtype=torch.float32), torch.tensor(view_one_hot, dtype=torch.float32)\n            \ntrain_transform = A.Compose([\n    A.Resize(Config.IMG_SIZE, Config.IMG_SIZE),\n    A.HorizontalFlip(p=0.6),\n    A.Rotate(limit=15, p=0.5),\n    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, p=0.4),\n    A.GridDistortion(\n        num_steps=5,\n        distort_limit=0.2,\n        p=0.5\n    ),\n    A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n    ToTensorV2(),\n])\n\nval_transform = A.Compose([\n    A.Resize(Config.IMG_SIZE, Config.IMG_SIZE),\n    A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n    ToTensorV2(),\n])\n# --- 6. Model ---\n\ndef get_model():\n    model = eva_x_base_patch16(pretrained='/kaggle/input/eva-x-base-pos/evax_base_pos_fold0.pth')\n    # model = eva_x_small_patch16(pretrained='eva_x_small_patch16_merged520k_mim.pt')\n    in_features = model.head.in_features\n    model.head = nn.Linear(in_features, Config.NUM_CLASSES)  \n    # model = timm.create_model(Config.MODEL_NAME, pretrained=True, num_classes=Config.NUM_CLASSES, drop_rate=Config.DROP_RATE, drop_path_rate=Config.DROP_PATH_RATE)\n    return model.to(Config.DEVICE)\n\n# --- 7. Training Loop (with EMA + AMP) ---\ndef run_training(model, train_loader, val_loader, criterion, ce_criterion, optimizer, scheduler, epochs, save_path, fold):\n    best_auc = 0\n    scaler = GradScaler()\n    ema = ModelEMA(model)\n    log_file = f\"training_log_pos_fold{fold}.txt\"\n    \n    for epoch in range(epochs):\n        model.train()\n        train_loss = 0\n        \n        for images, labels, view_one_hot in tqdm(train_loader, desc=f\"Training Epoch {epoch+1}/{epochs}\"):\n            images = images.to(Config.DEVICE)\n            labels = labels.to(Config.DEVICE)\n            view_one_hot = view_one_hot.to(Config.DEVICE)\n            # images, labels = mixup_cutmix(\n            #     images, labels,\n            #     mixup_alpha=0.2,\n            #     cutmix_alpha=0.1,\n            #     prob=0.5,        \n            #     switch_prob=0.5 \n            # )\n            optimizer.zero_grad()\n            with autocast():\n                outputs, out4 = model(images, out4=True)\n                base_loss = criterion(outputs, labels)\n                loss4 = ce_criterion(out4, view_one_hot)\n                loss = base_loss + 0.1 * loss4\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            train_loss += loss.item() * images.size(0)\n            ema.update(model)\n\n        model.eval()\n        val_loss = 0\n        all_labels_14, all_preds_14, all_labels_4, all_preds_4 = [], [], [], []\n        with torch.no_grad():\n            for images, labels, view_one_hot in tqdm(val_loader, desc=f\"Validating Epoch {epoch+1}/{epochs}\"):\n                images = images.to(Config.DEVICE)\n                labels = labels.to(Config.DEVICE)\n                view_one_hot = view_one_hot.to(Config.DEVICE)\n                outputs, out4 = ema.ema(images, out4=True)\n                loss = criterion(outputs, labels)\n                val_loss += loss.item() * images.size(0)\n                all_labels_14.append(labels.cpu().numpy())\n                all_preds_14.append(torch.sigmoid(outputs).cpu().numpy())\n                all_labels_4.append(view_one_hot.cpu().numpy())\n                all_preds_4.append(F.softmax(out4, dim=1).cpu().numpy())\n\n        train_loss /= len(train_loader.dataset)\n        val_loss /= len(val_loader.dataset)\n        all_labels_14 = np.concatenate(all_labels_14)\n        all_preds_14 = np.concatenate(all_preds_14)\n        val_auc_14 = roc_auc_score(all_labels_14, all_preds_14, average='macro')\n        per_class_auc_14 = []\n        for i, label_name in enumerate(Config.TARGET_LABELS):\n            try:\n                auc = roc_auc_score(all_labels_14[:, i], all_preds_14[:, i])\n                per_class_auc_14.append(auc)\n            except ValueError:\n                per_class_auc_14.append(np.nan)\n        all_labels_4 = np.concatenate(all_labels_4)\n        all_preds_4 = np.concatenate(all_preds_4)\n        val_auc_4 = roc_auc_score(all_labels_4, all_preds_4, multi_class='ovr', average='macro')\n        per_class_auc_4 = []\n        for i, label_name in enumerate(Config.VIEW_POSITION_LABELS):\n            try:\n                auc = roc_auc_score(all_labels_4[:, i], all_preds_4[:, i])\n                per_class_auc_4.append(auc)\n            except ValueError:\n                per_class_auc_4.append(np.nan)\n\n        with open(log_file, 'a') as f:\n            f.write(f\"\\nEpoch {epoch+1} | Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val Macro AUC (14 classes): {val_auc_14:.4f} | Val Macro AUC (View Positions): {val_auc_4:.4f}\\n\")\n            f.write(\"Per-class AUC scores (14 classes):\\n\")\n            for label_name, auc in zip(Config.TARGET_LABELS, per_class_auc_14):\n                f.write(f\"  {label_name}: {auc:.4f}\\n\" if not np.isnan(auc) else f\"  {label_name}: NaN (insufficient data)\\n\")\n            f.write(\"Per-class AUC scores (View Positions):\\n\")\n            for label_name, auc in zip(Config.VIEW_POSITION_LABELS, per_class_auc_4):\n                f.write(f\"  {label_name}: {auc:.4f}\\n\" if not np.isnan(auc) else f\"  {label_name}: NaN (insufficient data)\\n\")\n\n        print(f\"Epoch {epoch+1}/{epochs} | Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val Macro AUC (14 classes): {val_auc_14:.4f} | Val Macro AUC (View Positions): {val_auc_4:.4f} | Metrics saved to {log_file}\")\n\n        scheduler.step(val_auc_14)\n        if val_auc_14 > best_auc:\n            best_auc = val_auc_14\n            torch.save(ema.ema.state_dict(), save_path)\n            with open(log_file, 'a') as f:\n                f.write(f\"New best model saved with Macro AUC (14 classes): {best_auc:.4f} at {save_path}\\n\")\n            print(f\"New best model saved with Macro AUC (14 classes): {best_auc:.4f} at {save_path}\")\n    return best_auc\n\n\n# # --- 8. Stage 1: Training with Stratified K-Fold ---\n# print(\"\\n\" + \"=\"*20 + \" STAGE 1: Initial Stratified K-Fold Training \" + \"=\"*20)\n# initial_model_paths = []\n# for fold in range(Config.NUM_FOLDS):\n#     print(f\"\\nTraining Fold {fold+1}/{Config.NUM_FOLDS}\")\n#     train_idx, val_idx = fold_indices[fold]\n#     train_split_df = train_df.iloc[train_idx].reset_index(drop=True)\n#     val_split_df = train_df.iloc[val_idx].reset_index(drop=True)\n#     train_loader = DataLoader(\n#         ChestXRayDataset(train_split_df, transform=train_transform),\n#         batch_size=Config.BATCH_SIZE,\n#         shuffle=True,\n#         num_workers=Config.NUM_WORKERS\n#     )\n#     val_loader = DataLoader(\n#         ChestXRayDataset(val_split_df, transform=val_transform),\n#         batch_size=Config.BATCH_SIZE,\n#         shuffle=False,\n#         num_workers=Config.NUM_WORKERS\n#     )\n#     model = get_model()\n#     class_frequencies = train_df[Config.TARGET_LABELS].mean().values\n#     bce_weights = torch.tensor(1.0 / (class_frequencies + 1e-6)).to(Config.DEVICE)\n#     criterion = AsymmetricLossOptimized(gamma_neg=1.2, gamma_pos=2.5, clip=0.05)\n#     # criterion = FocalZLPR(tau=0.4, reduction='sum')\n#     # criterion = HybridLoss(\n#     #     bce_weights=bce_weights,\n#     #     alpha=0.1,\n#     #     beta=1,\n#     #     gamma_neg=1.2,\n#     #     gamma_pos=3.1,\n#     #     clip=0.05,\n#     #     eps=1e-8\n#     # )\n#     ce_criterion = nn.CrossEntropyLoss()\n#     # optimizer = optim.AdamW(model.parameters(), lr=Config.LEARNING_RATE)\n#     head_params = list(model.head.parameters()) + list(model.head4.parameters())\n\n#     optimizer = torch.optim.AdamW([\n#         {'params': [p for p in model.parameters() if id(p) not in list(map(id, head_params))],\n#         'lr': Config.LEARNING_RATE, 'weight_decay': 1e-4},\n#         {'params': head_params,\n#         'lr': Config.LEARNING_RATE * 10, 'weight_decay': 1e-4}\n#     ])\n#     scheduler = optim.lr_scheduler.OneCycleLR(\n#         optimizer,\n#         max_lr=Config.LEARNING_RATE * 10,\n#         epochs=Config.EPOCHS_INITIAL,\n#         steps_per_epoch=len(train_loader),\n#         pct_start=0.4,\n#         anneal_strategy='cos',\n#         div_factor=25.0,\n#         final_div_factor=1e4\n#     )\n#     save_path = f\"initial_best_model_pos_fold{fold}.pth\"\n#     run_training(model, train_loader, val_loader, criterion, ce_criterion, optimizer, scheduler, Config.EPOCHS_INITIAL, save_path, fold)\n#     initial_model_paths.append(save_path)\n\n# --- FINAL INFERENCE with Ensemble ---\nprint(\"\\n\" + \"=\"*20 + \" FINAL INFERENCE with Ensemble \" + \"=\"*20)\nfinal_models = []\ninitial_model_paths= ['/kaggle/input/eva-x-base-pos/evax_base_pos_fold0.pth',\n                     '/kaggle/input/eva-x-base-pos/evax_base_pos_fold1.pth',\n                     '/kaggle/input/eva-x-base-pos/evax_base_pos_fold2.pth',\n                     '/kaggle/input/eva-x-base-pos/evax_base_pos_fold3.pth',\n                     '/kaggle/input/eva-x-base-pos/evax_base_pos_fold4.pth']\nfor path in initial_model_paths:\n    model = get_model()\n    model.load_state_dict(torch.load(path))\n    model.eval()\n    final_models.append(model)\n\nfinal_test_df = sample_submission_df.sample(n=Config.DEBUG_FINAL_INFERENCE_SIZE, random_state=42) if Config.DEBUG else sample_submission_df\nif Config.DEBUG: print(f\"Debug mode ON: Running final inference on {len(final_test_df)} samples.\")\n\nfinal_test_loader = DataLoader(ChestXRayDataset(final_test_df, val_transform, is_test=True), batch_size=Config.BATCH_SIZE, shuffle=False, num_workers=Config.NUM_WORKERS)\nfinal_preds = []\nwith torch.no_grad():\n    for images in tqdm(final_test_loader, desc=\"Final Predicting\"):\n        avg_outputs = torch.zeros((images.size(0), Config.NUM_CLASSES), device=Config.DEVICE)\n        for model in final_models:\n            outputs = model(images.to(Config.DEVICE))\n            avg_outputs += outputs / Config.NUM_FOLDS\n        final_preds.append(torch.sigmoid(avg_outputs).cpu().numpy())\n\npredictions = np.concatenate(final_preds)\nsubmission_df = pd.DataFrame(predictions, columns=Config.TARGET_LABELS)\nsubmission_df[Config.IMAGE_COLUMN_NAME] = final_test_df[Config.IMAGE_COLUMN_NAME].values\nsubmission_df = submission_df[[Config.IMAGE_COLUMN_NAME] + Config.TARGET_LABELS]\nsubmission_df.to_csv(\"submission.csv\", index=False)\n\nprint(\"Submission file created successfully!\")\nprint(submission_df.head())\nprint(\"\\nAll fold models saved for ensemble:\")\nfor path in initial_model_paths:\n    print(path)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-09T09:05:54.331151Z","iopub.execute_input":"2025-10-09T09:05:54.331478Z","iopub.status.idle":"2025-10-09T09:15:57.818084Z","shell.execute_reply.started":"2025-10-09T09:05:54.331442Z","shell.execute_reply":"2025-10-09T09:15:57.817072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}