{"cells":[{"metadata":{},"cell_type":"markdown","source":"# importing libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"timm_path = \"../input/timm-pytorch-image-models/pytorch-image-models-master\"\nimport sys\nsys.path.append(timm_path)\nimport timm\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport sklearn\nimport os\nfrom tqdm.notebook import tqdm\n\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch import optim\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train  =  pd.read_csv(\"../input/vinbigdata-chest-xray-abnormalities-detection/train.csv\") \nis_normal_df = train.groupby(\"image_id\")[\"class_id\"].agg(lambda s: (s == 14).sum()).reset_index().rename({\"class_id\": \"num_normal_annotations\"}, axis=1)\nis_normal_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To understand this piece of code above go to the awesome notebook of @corochann https://www.kaggle.com/corochann/vinbigdata-2-class-classifier-complete-pipeline/notebook, this peice of code was taken from his notebook."},{"metadata":{"trusted":true},"cell_type":"code","source":"def change(x):\n    if (x==3):\n        x=1\n    return x\nis_normal_df['target'] = is_normal_df['num_normal_annotations'].apply(lambda x: change(x))\ndf = is_normal_df[[\"image_id\",\"target\"]]\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# splitting"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nskf  =  StratifiedKFold(n_splits = 5, random_state = 42,shuffle = True)\nfolds = df.copy()\nfor f,(tr_idx,val_idx) in enumerate(skf.split(folds,folds.target)):\n    folds.loc[val_idx,'fold'] = int(f)\nfolds['fold'] = folds['fold'].astype(int)    \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds.image_id=folds.image_id+\".png\"\nimg_path = \"../input/vinbigdata-chest-xray-resized-png-1024x1024/train\"\ndf_paths = [os.path.join(img_path,x) for x in folds.image_id]\nfolds['path'] = df_paths\nfolds.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_aug = A.Compose(\n    [  \n\n        A.Resize(300,300,p=1.0),\n        A.CLAHE(clip_limit=4.0, p=0.85),\n\n     A.Normalize(\n            p=1.0),\n        ToTensorV2(p=1.0)\n    ]\n)\nval_aug = A.Compose(\n    [\n         A.Resize(300,300,p=1.0),\n        A.HorizontalFlip(p=0.5),\n         A.Normalize(\n            p=1.0),\n        ToTensorV2(p=1.0),\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Xray(Dataset):\n    def __init__(self,df,augs=None):\n        self.df = df\n        self.augs = augs\n    def __len__(self):\n        return(len(self.df))\n    def __getitem__(self,idx):\n        img_src = self.df.loc[idx,'path']\n        image = cv2.imread(img_src)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.uint8)\n        \n        target = self.df.loc[idx,'target']\n        \n        if (self.augs):\n            transformed = self.augs(image=image)\n            image = transformed['image']\n        \n        return image,torch.tensor(target) \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = Xray(folds,augs = train_aug)\nload = DataLoader(data,batch_size = 1)\nimg,target = next(iter(load))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(img.squeeze(0).permute(1,2,0))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Efficientnet b3 noisy student Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model=timm.create_model('tf_efficientnet_b3_ns', pretrained=False) # set pretrained=True to use the pretrained weights\nnum_features = model.classifier.in_features\nmodel.classifier = nn.Linear(num_features, 1)\nmodel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ss=F.sigmoid(model(torch.randn(3,3,300,300)))\nss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helping Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_epoch(train_loader,model,optimizer,criterion,e,epochs):\n    losses = AverageMeter()\n    scores = AverageMeter()\n    model.train()\n    global_step = 0\n    loop = tqdm(enumerate(train_loader),total = len(train_loader))\n    \n    for step,(image,labels) in loop:\n        image = image.to(device)\n        labels = labels.unsqueeze(1)\n        labels= labels.to(device)\n        output = model(image)\n        batch_size = labels.size(0)\n        loss = criterion(output,labels.float())\n        \n        out = F.sigmoid(output)\n        outputs = out.cpu().detach().numpy()\n        targets = labels.cpu().detach().numpy()\n        try:\n            auc = sklearn.metrics.roc_auc_score(targets, outputs)\n            losses.update(loss.item(), batch_size)\n            scores.update(auc.item(), batch_size)\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n            global_step += 1\n        \n            loop.set_description(f\"Epoch {e+1}/{epochs}\")\n            loop.set_postfix(loss = loss.item(), auc = auc.item(), stage = 'train')\n        \n            \n        except ValueError:\n            pass\n        \n        \n       \n        \n    return losses.avg,scores.avg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def val_one_epoch(loader,model,optimizer,criterion):\n    losses = AverageMeter()\n    scores = AverageMeter()\n    model.eval()\n    global_step = 0\n    loop = tqdm(enumerate(loader),total = len(loader))\n    \n    for step,(image,labels) in loop:\n        image = image.to(device)\n        labels = labels.unsqueeze(1)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        with torch.no_grad():\n            output = model(image)\n        loss = criterion(output,labels.float())\n        \n        out = F.sigmoid(output)\n        outputs = out.cpu().detach().numpy()\n        targets = labels.cpu().detach().numpy()\n        try:\n            auc = sklearn.metrics.roc_auc_score(targets, outputs)\n            losses.update(loss.item(), batch_size)\n            scores.update(auc.item(), batch_size)\n            loop.set_postfix(loss = loss.item(), auc = auc.item(), stage = 'valid')\n            optimizer.step()\n            optimizer.zero_grad()\n            global_step += 1\n        except ValueError:\n            pass\n        \n        \n        \n        \n        \n    \n        \n    return losses.avg,scores.avg","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit(model,fold_n,training_batch_size=32,validation_batch_size=64):\n    \n    train_data=folds[folds.fold != fold_n]\n    val_data=folds[folds.fold == fold_n]\n    train_data= Xray(train_data.reset_index(drop=True),augs = train_aug)\n    val_data= Xray(val_data.reset_index(drop=True),augs = val_aug)\n    \n    \n    train_loader = DataLoader(train_data,\n                             shuffle=True,\n                        num_workers=0,\n                        batch_size=training_batch_size)\n    valid_loader = DataLoader(val_data,\n                             shuffle=False,\n                        num_workers=0,\n                        batch_size=validation_batch_size)\n    model = model\n    model.to(device)\n    criterion=nn.BCEWithLogitsLoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.001)\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience = 3,verbose = True)\n    epochs= 5\n    \n    best_acc = 0\n    \n    loop = range(epochs)\n    for e in loop:\n        \n        train_loss,train_auc = train_one_epoch(train_loader,model,optimizer,criterion,e,epochs)\n         #scheduling step if given\n    \n        #scheduler.step()\n        \n        print(f'For epoch {e+1}/{epochs}')\n        print(f'average train_loss {train_loss}')\n        print(f'average train_auc {train_auc}' )\n        \n        val_loss,val_auc = val_one_epoch(valid_loader,model,optimizer,criterion)\n        \n        scheduler.step(val_loss)\n        \n        print(f'avarage val_loss { val_loss }')\n        print(f'avarage val_auc {val_auc}')\n        \n        \n        \n        \n        if (val_auc>best_acc):\n            best_acc =val_auc\n            print(f'saving model for {best_acc}')\n            torch.save(model.state_dict(),OUTPUT_DIR+ f'Fold {fold_n} model with val_acc {best_acc}.pth') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fit(model,0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Not training further epochs due to GPU constraints\n#To get a idea of this full pipeline, head over to https://www.kaggle.com/mrinath/another-simple-and-fast-pytorch-pipeline\n#say_my_name"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}