{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# setting kernel type\n\nto use tpu  set training = 'tpu'   for gpu  training = 'gpu'   for cpu training = 'cpu'\n\n# **Note for TPU\n**while these notebook is ok to work in tpu  ,  the xla for pytorch As per they have announced, its still under development and so for operations not supported they automatically switch to CPU so it might get even slower than GPU.\n\nso for some reson in these notebook when using this data sets where  we have  large size of images when  image resize and transformations  it will use cpu for that transformation and do training in tpu and these will cost memory to work around with these either you save images after transformation and read them diractly into the model or  i just skip val phase and also  make batches very small in tpu \nto  use tpu in these dataset\njust set profile='low'  other  can set profile=''high'","metadata":{}},{"cell_type":"code","source":"training = 'gpu'\nprofile =  'high'  # for this dtata set to use tpu set these var to low\norginal = False # work on orginal data or on the preprocessed jpg data","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:42:47.435333Z","iopub.execute_input":"2022-08-11T14:42:47.436467Z","iopub.status.idle":"2022-08-11T14:42:47.540948Z","shell.execute_reply.started":"2022-08-11T14:42:47.436321Z","shell.execute_reply":"2022-08-11T14:42:47.539943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# v9 using mayo-clinic-strip-ai-competition-1k-png-data","metadata":{}},{"cell_type":"markdown","source":"# install lighting and xla for tpu","metadata":{}},{"cell_type":"code","source":"if training =='tpu':\n    !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n    !python pytorch-xla-env-setup.py --version 1.7 --apt-packages libomp5 libopenblas-dev","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:42:51.914011Z","iopub.execute_input":"2022-08-11T14:42:51.914907Z","iopub.status.idle":"2022-08-11T14:42:51.922321Z","shell.execute_reply.started":"2022-08-11T14:42:51.914877Z","shell.execute_reply":"2022-08-11T14:42:51.921365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install pytorch-lightning==1.1.8\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-11T14:42:52.83207Z","iopub.execute_input":"2022-08-11T14:42:52.832754Z","iopub.status.idle":"2022-08-11T14:43:19.385261Z","shell.execute_reply.started":"2022-08-11T14:42:52.832725Z","shell.execute_reply":"2022-08-11T14:43:19.383973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:19.389214Z","iopub.execute_input":"2022-08-11T14:43:19.389569Z","iopub.status.idle":"2022-08-11T14:43:29.737614Z","shell.execute_reply.started":"2022-08-11T14:43:19.389496Z","shell.execute_reply":"2022-08-11T14:43:29.736477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install rasterio","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:29.739755Z","iopub.execute_input":"2022-08-11T14:43:29.740131Z","iopub.status.idle":"2022-08-11T14:43:38.628468Z","shell.execute_reply.started":"2022-08-11T14:43:29.740056Z","shell.execute_reply":"2022-08-11T14:43:38.626961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# library  needed by lightining","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.nn import functional as F\nfrom torch import nn\nfrom pytorch_lightning.core import LightningModule\nfrom sklearn.metrics import label_ranking_average_precision_score\n\nprint(torch.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:38.632133Z","iopub.execute_input":"2022-08-11T14:43:38.632719Z","iopub.status.idle":"2022-08-11T14:43:46.312416Z","shell.execute_reply.started":"2022-08-11T14:43:38.632662Z","shell.execute_reply":"2022-08-11T14:43:46.311049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# other libraarys","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\nfrom torchvision import datasets, models, transforms\nfrom torch import nn, optim\nfrom torch.optim import lr_scheduler\nfrom torch.autograd import Variable\nfrom torchvision import datasets, models, transforms  \nfrom torch.utils.data.sampler import SubsetRandomSampler  \nfrom torch.utils.data import Dataset, DataLoader\n\nimport pandas as pd\nimport torch.nn.functional as F\n\nimport time\nimport os\nimport time\nimport random\nfrom datetime import datetime\nimport tifffile\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom sklearn import model_selection, metrics\nimport cv2\nimport timm\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint , EarlyStopping\nfrom PIL import Image, ImageSequence\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:46.318524Z","iopub.execute_input":"2022-08-11T14:43:46.322124Z","iopub.status.idle":"2022-08-11T14:43:47.158536Z","shell.execute_reply.started":"2022-08-11T14:43:46.322037Z","shell.execute_reply":"2022-08-11T14:43:47.157481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# seeding everthing","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n    \n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if training == 'gpu':\n        torch.cuda.manual_seed(seed)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\nseed_everything(3407)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:47.160482Z","iopub.execute_input":"2022-08-11T14:43:47.160823Z","iopub.status.idle":"2022-08-11T14:43:47.171796Z","shell.execute_reply.started":"2022-08-11T14:43:47.160783Z","shell.execute_reply":"2022-08-11T14:43:47.17045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# configartions","metadata":{}},{"cell_type":"code","source":"DATA_PATH = \"../input/mayo-clinic-strip-ai/\"\n\nif orginal:\n    TRAIN_PATH = \"../input/mayo-clinic-strip-ai/train/\"\nelse:\n    TRAIN_PATH = \"../input/mayo-clinic-strip-ai-competition-1k-png-data/\"\nTEST_PATH = \"../input/mayo-clinic-strip-ai/test/\"\n\n\n\n\nIMG_SIZE = 384\nif profile =='low':\n    BATCH_SIZE = 2 \n    val_BATCH_SIZE = 2\n    valchecking = 0\n    epoch = 3\n    monit = 'val_loss'\nelse:\n    BATCH_SIZE = 8\n    val_BATCH_SIZE = 8\n    valchecking = 1\n    epoch = 10\n    monit = 'val_loss'","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:48.027511Z","iopub.execute_input":"2022-08-11T14:43:48.027798Z","iopub.status.idle":"2022-08-11T14:43:48.034871Z","shell.execute_reply.started":"2022-08-11T14:43:48.02777Z","shell.execute_reply":"2022-08-11T14:43:48.03373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# read csv file with pandas","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(os.path.join(DATA_PATH, \"train.csv\"))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:53.106063Z","iopub.execute_input":"2022-08-11T14:43:53.106922Z","iopub.status.idle":"2022-08-11T14:43:53.152988Z","shell.execute_reply.started":"2022-08-11T14:43:53.106891Z","shell.execute_reply":"2022-08-11T14:43:53.15189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:54.611255Z","iopub.execute_input":"2022-08-11T14:43:54.611535Z","iopub.status.idle":"2022-08-11T14:43:54.63576Z","shell.execute_reply.started":"2022-08-11T14:43:54.611507Z","shell.execute_reply":"2022-08-11T14:43:54.634459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:56.122076Z","iopub.execute_input":"2022-08-11T14:43:56.122801Z","iopub.status.idle":"2022-08-11T14:43:56.135219Z","shell.execute_reply.started":"2022-08-11T14:43:56.122771Z","shell.execute_reply":"2022-08-11T14:43:56.133812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.label.value_counts().plot(kind=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:57.306079Z","iopub.execute_input":"2022-08-11T14:43:57.307027Z","iopub.status.idle":"2022-08-11T14:43:57.820459Z","shell.execute_reply.started":"2022-08-11T14:43:57.306968Z","shell.execute_reply":"2022-08-11T14:43:57.817293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:43:58.446993Z","iopub.execute_input":"2022-08-11T14:43:58.447406Z","iopub.status.idle":"2022-08-11T14:43:58.477455Z","shell.execute_reply.started":"2022-08-11T14:43:58.447369Z","shell.execute_reply":"2022-08-11T14:43:58.476632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# split data","metadata":{}},{"cell_type":"code","source":"train_df, valid_df = model_selection.train_test_split(\n    df, test_size=0.1, random_state=999\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:44:00.832232Z","iopub.execute_input":"2022-08-11T14:44:00.832719Z","iopub.status.idle":"2022-08-11T14:44:00.842379Z","shell.execute_reply.started":"2022-08-11T14:44:00.832673Z","shell.execute_reply":"2022-08-11T14:44:00.841303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data set class for getting data from path","metadata":{}},{"cell_type":"code","source":"\n    \nclass TaskDataset(torch.utils.data.Dataset):\n    \"\"\"\n    Helper Class to create the pytorch dataset\n    \"\"\"\n\n    def __init__(self, df, data_path=DATA_PATH, mode=\"train\",  transforms=None):\n        super().__init__()\n        self.df_data = df\n        self.data_path = data_path\n        self.transforms = transforms\n        self.mode = mode\n        self.data_dir = TRAIN_PATH if mode == \"train\" else TEST_PATH\n\n    def __len__(self):\n        return len(self.df_data)\n\n    def __getitem__(self, index):\n        img_name = self.df_data.image_id.values[index]\n        if orginal:\n            img_path = os.path.join(self.data_dir, img_name + \".tif\" )\n            img  =  tifffile.imread(img_path)  \n            img = Image.fromarray(np.uint8(img)).resize((IMG_SIZE, IMG_SIZE))\n        else:\n            img_path = os.path.join(self.data_dir, img_name + \".png\" )\n            img =   Image.open(img_path).convert(\"RGB\")\n        if self.mode == \"train\":\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df_data.iloc[index].label]\n            label = torch.tensor(label,dtype= torch.float32) \n            label = label.type(torch.LongTensor) \n        if self.transforms is not None:\n            image = self.transforms(img)\n        if self.mode == \"train\":\n            return image, label\n        del img\n        return image","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:13:08.708249Z","iopub.execute_input":"2022-08-11T15:13:08.70856Z","iopub.status.idle":"2022-08-11T15:13:08.719916Z","shell.execute_reply.started":"2022-08-11T15:13:08.708531Z","shell.execute_reply":"2022-08-11T15:13:08.718594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **  transforming data doing augmention **","metadata":{}},{"cell_type":"code","source":"# Imagenet means and stds\nmean = [0.485, 0.456, 0.406]\nstd  = [0.229, 0.224, 0.225] \n        \ntransforms_train = transforms.Compose([\n                            transforms.Resize((IMG_SIZE, IMG_SIZE)),\n                         # transforms.RandomRotation(45),\n                           # transforms.RandomHorizontalFlip(),\n                           # transforms.RandomVerticalFlip(),\n                          #  transforms.RandomResizedCrop(IMG_SIZE),\n                            transforms.ToTensor(),\n                         # transforms.Normalize(mean=mean, std=std),\n                           # transforms.Grayscale(num_output_channels=1)\n                        ]\n                    )\n\ntransforms_valid = transforms.Compose(\n                        [\n                            transforms.Resize((IMG_SIZE, IMG_SIZE)),\n                            transforms.ToTensor(),\n                         # transforms.Normalize(mean=mean, std=std),\n                            #transforms.Grayscale(num_output_channels=1)\n                        ]\n                    )        \n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:13:10.298686Z","iopub.execute_input":"2022-08-11T15:13:10.299875Z","iopub.status.idle":"2022-08-11T15:13:10.308012Z","shell.execute_reply.started":"2022-08-11T15:13:10.299832Z","shell.execute_reply":"2022-08-11T15:13:10.306758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  make datasets and data loaders","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = TaskDataset(train_df, transforms=transforms_train)\nvalid_dataset = TaskDataset(valid_df, transforms=transforms_valid)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:13:38.311007Z","iopub.execute_input":"2022-08-11T15:13:38.312242Z","iopub.status.idle":"2022-08-11T15:13:38.318271Z","shell.execute_reply.started":"2022-08-11T15:13:38.312166Z","shell.execute_reply":"2022-08-11T15:13:38.317193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''del df\ndel train_df\ndel valid_df'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:13:40.087432Z","iopub.execute_input":"2022-08-11T15:13:40.087776Z","iopub.status.idle":"2022-08-11T15:13:40.097536Z","shell.execute_reply.started":"2022-08-11T15:13:40.087745Z","shell.execute_reply":"2022-08-11T15:13:40.096317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader =  DataLoader( dataset=train_dataset,\n        batch_size=BATCH_SIZE, \n      \n        drop_last=True,\n         shuffle=True)\nvalid_loader =    DataLoader(  dataset=valid_dataset,\n        batch_size=val_BATCH_SIZE ,\n        \n        drop_last=True,\n         shuffle=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:13:41.598739Z","iopub.execute_input":"2022-08-11T15:13:41.599635Z","iopub.status.idle":"2022-08-11T15:13:41.606646Z","shell.execute_reply.started":"2022-08-11T15:13:41.599602Z","shell.execute_reply":"2022-08-11T15:13:41.605564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs, classes = next(iter(train_loader))  \nprint(classes)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:13:45.968746Z","iopub.execute_input":"2022-08-11T15:13:45.97154Z","iopub.status.idle":"2022-08-11T15:13:46.454909Z","shell.execute_reply.started":"2022-08-11T15:13:45.971507Z","shell.execute_reply":"2022-08-11T15:13:46.453908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(inputs[0].shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:13:54.60892Z","iopub.execute_input":"2022-08-11T15:13:54.609253Z","iopub.status.idle":"2022-08-11T15:13:54.615579Z","shell.execute_reply.started":"2022-08-11T15:13:54.609208Z","shell.execute_reply":"2022-08-11T15:13:54.614502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.imshow(inputs[0].permute(1, 2, 0) )\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:14:03.148624Z","iopub.execute_input":"2022-08-11T15:14:03.148914Z","iopub.status.idle":"2022-08-11T15:14:03.446326Z","shell.execute_reply.started":"2022-08-11T15:14:03.148885Z","shell.execute_reply":"2022-08-11T15:14:03.445318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# in these code we defin pytorchlightning model\n\n\n","metadata":{}},{"cell_type":"code","source":"gpu = torch.cuda.is_available()\ndevice = torch.device(\"cuda\" if gpu else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:14:23.239458Z","iopub.execute_input":"2022-08-11T15:14:23.239778Z","iopub.status.idle":"2022-08-11T15:14:23.358207Z","shell.execute_reply.started":"2022-08-11T15:14:23.239748Z","shell.execute_reply":"2022-08-11T15:14:23.356357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_lightning.loggers import TensorBoardLogger\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:14:23.986703Z","iopub.execute_input":"2022-08-11T15:14:23.987011Z","iopub.status.idle":"2022-08-11T15:14:23.992398Z","shell.execute_reply.started":"2022-08-11T15:14:23.986982Z","shell.execute_reply":"2022-08-11T15:14:23.990787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg = []\ntr_avg = []","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:37:09.665389Z","iopub.execute_input":"2022-08-11T15:37:09.665743Z","iopub.status.idle":"2022-08-11T15:37:09.672216Z","shell.execute_reply.started":"2022-08-11T15:37:09.665712Z","shell.execute_reply":"2022-08-11T15:37:09.670347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Mamonmodel(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        i = 0\n        self.lr=0.0003\n        self.wd = 1e-6\n        self.schd = 'ReduceLROnPlateau'\n        #self.best_model_wts = copy.deepcopy(model.state_dict())\n        self.best_loss = np.Inf\n        #self.logger = TensorBoardLogger()\n        basemodel = torch.hub.load('facebookresearch/deit:main', 'deit_base_patch16_384', pretrained=True)\n        for child in basemodel.blocks.children():\n            if i < 5:\n                for param in child.parameters():\n                    param.requires_grad = False\n            else:\n                for param in child.parameters():\n                    param.requires_grad = True\n            i +=1\n        print(i)\n        basemodel.head  =  nn.Sequential( \n                            \n             nn.Linear(768, 2)  \n                    )\n        self.model = basemodel\n        \n    def forward(self,x):\n        x = self.model(x)\n        return x\n    def training_step(self,batch,batch_idx):\n        x , y = batch\n        y_hat = self.model(x)\n        loss =  F.cross_entropy(y_hat, y) # F.binary_cross_entropy_with_logits(y_hat , y)\n        #scor = label_ranking_average_precision_score(y.cpu(), torch.sigmoid(y_hat).cpu())\n        self.log('train_loss', loss)\n        #self.log('train_score', scor)\n        return  loss   \n    def validation_step(self,batch,batch_idx):\n        x , y = batch\n        y_hat = self.model(x)\n        loss = F.cross_entropy(y_hat, y) # F.binary_cross_entropy_with_logits(y_hat , y)\n        #scor = label_ranking_average_precision_score(y.cpu(), torch.sigmoid(y_hat).cpu())\n        #self.log('val_score', scor)\n        self.log('val_loss', loss) \n        if self.best_loss > loss:\n            self.best_loss = loss\n            torch.save(self.model.state_dict(), 'Deit-bestwigh-ep-'+str(self.current_epoch)+'.pth')\n            torch.save(self.model, 'Deit-best-model.pth')\n        return {'val_loss': loss}\n    def training_epoch_end(self, outputs):\n        #print(outputs)\n        avg_loss = torch.stack([x['loss'] for x in outputs]).mean()\n        self.logger.experiment.add_scalar('train_loss',avg_loss, self.current_epoch)\n        tr_avg.append(avg_loss)\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n        self.logger.experiment.add_scalar('loss',avg_loss, self.current_epoch)\n        avg.append(avg_loss)\n        return {'av_loss' : avg_loss}\n    def configure_optimizers(self):\n        optimizer = optim.AdamW(\n            self.model.parameters(), lr=self.lr, weight_decay=self.wd\n        )\n        if self.schd == 'ReduceLROnPlateau':\n            optimizer = optim.Adam(\n            self.model.parameters(), lr=self.lr \n        )\n            scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.9, patience=3,\n                                          verbose=True)\n        elif self.schd == 'CosineAnnealingLR':\n            scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0)\n        elif self.schd == 'CosineAnnealingWarmRestarts':\n            scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, eta_min=0.001, last_epoch=-1) \n\n        return {'optimizer': optimizer, 'lr_scheduler': scheduler  ,  'monitor':monit }\n        ","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:37:11.229331Z","iopub.execute_input":"2022-08-11T15:37:11.229714Z","iopub.status.idle":"2022-08-11T15:37:11.249824Z","shell.execute_reply.started":"2022-08-11T15:37:11.229654Z","shell.execute_reply":"2022-08-11T15:37:11.248646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# init model\nmemomodel = Mamonmodel()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:37:13.623892Z","iopub.execute_input":"2022-08-11T15:37:13.624643Z","iopub.status.idle":"2022-08-11T15:37:16.102206Z","shell.execute_reply.started":"2022-08-11T15:37:13.624613Z","shell.execute_reply":"2022-08-11T15:37:16.101034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n{val_loss:.2f}\n'''\nearly_stop_callback = EarlyStopping(\n   monitor='val_loss',\n    min_delta=0.0000,\n   patience=5,\n    verbose=True,\n   mode='min'\n )\nif training == 'gpu':\n    checkpoint_callback = ModelCheckpoint(filename='{epoch}-gpumodel.pth', \n    verbose=True,\n    monitor='val_loss',\n    mode='min')\n    trainer = pl.Trainer(min_epochs=1 , max_epochs=epoch, check_val_every_n_epoch=valchecking,\n            \n                        gpus=-1,callbacks=[checkpoint_callback , early_stop_callback]\n    ) \n    \nelif training == 'tpu':\n    print('now we n tpu')\n    checkpoint_callback = ModelCheckpoint(filename='{epoch}-tpumodel.pth',  \n    verbose=True,\n    monitor=monit,\n    mode='min')\n    trainer = pl.Trainer(tpu_cores=8,min_epochs=1,num_sanity_val_steps=0 , check_val_every_n_epoch=valchecking , max_epochs=epoch,  callbacks=[checkpoint_callback])\nelse:\n    checkpoint_callback = ModelCheckpoint(filename='{epoch}-cpumodel.pth',  \n    verbose=True,\n    monitor='val_loss',\n    mode='min')\n    trainer = pl.Trainer(min_epochs=1 , max_epochs=epoch,check_val_every_n_epoch=valchecking,  callbacks=[checkpoint_callback , early_stop_callback])\n\n    \n\n\ntrainer.fit(memomodel, train_loader , valid_loader)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:37:23.539536Z","iopub.execute_input":"2022-08-11T15:37:23.539832Z","iopub.status.idle":"2022-08-11T15:49:14.469535Z","shell.execute_reply.started":"2022-08-11T15:37:23.539804Z","shell.execute_reply":"2022-08-11T15:49:14.468466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = {'val_avg' : [ x.item() for x in  avg[1:] ]  , 'train_avg ': [ xx.item() for xx in  tr_avg ] } \nout_res = pd.DataFrame.from_dict(all_data)\nout_res.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:49:14.47273Z","iopub.execute_input":"2022-08-11T15:49:14.473565Z","iopub.status.idle":"2022-08-11T15:49:14.500129Z","shell.execute_reply.started":"2022-08-11T15:49:14.47352Z","shell.execute_reply":"2022-08-11T15:49:14.499158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference ","metadata":{}},{"cell_type":"code","source":" MODEL_PATH = (\n    \"./Deit-best-model.pth\"\n)\nBATCH_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:40.555909Z","iopub.execute_input":"2022-08-09T11:29:40.556788Z","iopub.status.idle":"2022-08-09T11:29:40.561251Z","shell.execute_reply.started":"2022-08-09T11:29:40.556743Z","shell.execute_reply":"2022-08-09T11:29:40.56035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dftest = pd.read_csv(os.path.join(DATA_PATH, \"test.csv\"))\ndftest.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:41.35917Z","iopub.execute_input":"2022-08-09T11:29:41.35941Z","iopub.status.idle":"2022-08-09T11:29:41.388484Z","shell.execute_reply.started":"2022-08-09T11:29:41.359382Z","shell.execute_reply":"2022-08-09T11:29:41.38774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntransforms_test = transforms.Compose(\n    [\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:44.216312Z","iopub.execute_input":"2022-08-09T11:29:44.216893Z","iopub.status.idle":"2022-08-09T11:29:44.22361Z","shell.execute_reply.started":"2022-08-09T11:29:44.216855Z","shell.execute_reply":"2022-08-09T11:29:44.221071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PATH = '../input/jpg-images-strip-ai/test/'","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:44.999352Z","iopub.execute_input":"2022-08-09T11:29:44.999849Z","iopub.status.idle":"2022-08-09T11:29:45.004517Z","shell.execute_reply.started":"2022-08-09T11:29:44.999817Z","shell.execute_reply":"2022-08-09T11:29:45.003184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = TaskDataset(dftest,data_path=TEST_PATH,  mode=\"test\" , transforms=transforms_test )\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:45.777689Z","iopub.execute_input":"2022-08-09T11:29:45.778112Z","iopub.status.idle":"2022-08-09T11:29:45.784302Z","shell.execute_reply.started":"2022-08-09T11:29:45.778082Z","shell.execute_reply":"2022-08-09T11:29:45.783361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loader = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False, pin_memory=False) \n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:46.943319Z","iopub.execute_input":"2022-08-09T11:29:46.944058Z","iopub.status.idle":"2022-08-09T11:29:46.949301Z","shell.execute_reply.started":"2022-08-09T11:29:46.944023Z","shell.execute_reply":"2022-08-09T11:29:46.948427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"memomodel.eval()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:48.094885Z","iopub.execute_input":"2022-08-09T11:29:48.095168Z","iopub.status.idle":"2022-08-09T11:29:48.106557Z","shell.execute_reply.started":"2022-08-09T11:29:48.095137Z","shell.execute_reply":"2022-08-09T11:29:48.105779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"memomodel.to(device)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:49.63666Z","iopub.execute_input":"2022-08-09T11:29:49.63723Z","iopub.status.idle":"2022-08-09T11:29:49.796263Z","shell.execute_reply.started":"2022-08-09T11:29:49.637198Z","shell.execute_reply":"2022-08-09T11:29:49.795351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport tqdm.notebook as tq\npreds_torch = []\nwith torch.no_grad():\n    test_tqdm  = tq.tqdm(test_loader, total=len(test_loader), desc=\"Testing\", position=0, leave=True)\n    for images in test_tqdm:\n        images = images.to(device)\n        preds_torch.extend(memomodel(images).detach().cpu().numpy()) ","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:51.499767Z","iopub.execute_input":"2022-08-09T11:29:51.501732Z","iopub.status.idle":"2022-08-09T11:29:51.717252Z","shell.execute_reply.started":"2022-08-09T11:29:51.501673Z","shell.execute_reply":"2022-08-09T11:29:51.716465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_torch","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:54.448495Z","iopub.execute_input":"2022-08-09T11:29:54.448775Z","iopub.status.idle":"2022-08-09T11:29:54.460945Z","shell.execute_reply.started":"2022-08-09T11:29:54.448745Z","shell.execute_reply":"2022-08-09T11:29:54.459993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_out = pd.DataFrame(preds_torch, columns = ['CE','LAA'])\ndf_out","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:29:58.26595Z","iopub.execute_input":"2022-08-09T11:29:58.266807Z","iopub.status.idle":"2022-08-09T11:29:58.279186Z","shell.execute_reply.started":"2022-08-09T11:29:58.26676Z","shell.execute_reply":"2022-08-09T11:29:58.277497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:30:03.82497Z","iopub.execute_input":"2022-08-09T11:30:03.825265Z","iopub.status.idle":"2022-08-09T11:30:03.842034Z","shell.execute_reply.started":"2022-08-09T11:30:03.825232Z","shell.execute_reply":"2022-08-09T11:30:03.841275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['CE'] = df_out['CE']\nsub['LAA'] = df_out['LAA']\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:30:07.111134Z","iopub.execute_input":"2022-08-09T11:30:07.111551Z","iopub.status.idle":"2022-08-09T11:30:07.12943Z","shell.execute_reply.started":"2022-08-09T11:30:07.111518Z","shell.execute_reply":"2022-08-09T11:30:07.128611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index = False )\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:30:15.731215Z","iopub.execute_input":"2022-08-09T11:30:15.731544Z","iopub.status.idle":"2022-08-09T11:30:15.747625Z","shell.execute_reply.started":"2022-08-09T11:30:15.731513Z","shell.execute_reply":"2022-08-09T11:30:15.746892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}