{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### [Training NOtebook](https://www.kaggle.com/code/anantgupt/pyt-mayo-strip-ai-eda-train-efficientnet)","metadata":{}},{"cell_type":"code","source":"import gc\nimport os \n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport tifffile\nimport torch\nimport torch.nn as nn\n\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom torchvision import models, transforms\nfrom torch.nn.functional import softmax\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm import tqdm\n","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:32:34.852403Z","iopub.execute_input":"2022-09-27T05:32:34.852783Z","iopub.status.idle":"2022-09-27T05:32:37.048889Z","shell.execute_reply.started":"2022-09-27T05:32:34.852752Z","shell.execute_reply":"2022-09-27T05:32:37.047768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debug = False\ngenerate_new = True\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\ntest_df","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:32:41.289238Z","iopub.execute_input":"2022-09-27T05:32:41.28981Z","iopub.status.idle":"2022-09-27T05:32:41.320075Z","shell.execute_reply.started":"2022-09-27T05:32:41.289773Z","shell.execute_reply":"2022-09-27T05:32:41.31914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    os.mkdir(\"../test/\")\nexcept:\n    pass\nfor i in tqdm(range(test_df.shape[0])):\n    img_id = test_df.iloc[i].image_id\n    try:\n        sz = os.path.getsize(dirs[1] + img_id + \".tif\")\n    except:\n        sz = 1000000000\n    if(sz > 8e8):\n        img = np.zeros((512,512,3), np.uint8)\n    else:\n        try:\n            img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n        except:\n            img = np.zeros((512,512,3), np.uint8)\n    cv2.imwrite(f\"../test/{img_id}.jpg\", img)\n    del img\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:33:27.389006Z","iopub.execute_input":"2022-09-27T05:33:27.389433Z","iopub.status.idle":"2022-09-27T05:34:05.930345Z","shell.execute_reply.started":"2022-09-27T05:33:27.389391Z","shell.execute_reply":"2022-09-27T05:34:05.929346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"../test/\", \"../train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        try:\n            image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        except:\n            image = np.zeros((512,512,3), np.uint8)\n        label = 0\n        try:\n            if len(image.shape) == 5:\n                image = image.squeeze().transpose(1, 2, 0)\n            image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        except:\n            image = np.zeros((3, 512, 512))\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        patient_id = self.df.iloc[index].patient_id\n        return image, label, patient_id","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:35:08.947442Z","iopub.execute_input":"2022-09-27T05:35:08.947834Z","iopub.status.idle":"2022-09-27T05:35:08.960643Z","shell.execute_reply.started":"2022-09-27T05:35:08.947802Z","shell.execute_reply":"2022-09-27T05:35:08.959637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, dataloader):\n    model.cuda()\n    model.eval()\n    dataloader = dataloader\n    outputs = []\n    s = nn.Softmax(dim=1)\n    ids = []\n    for item in tqdm(dataloader, leave=False):\n        patient_id = item[2][0]\n        try:\n            images = item[0].cuda().float()\n            ids.append(patient_id)\n            output = model(images)\n            outputs.append(s(output.cpu()[:,:2])[0].detach().numpy())\n        except:\n            ids.append(patient_id)\n            outputs.append(s(torch.tensor([[1, 1]]).float())[0].detach().numpy())\n    return np.array(outputs), ids","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:35:21.805309Z","iopub.execute_input":"2022-09-27T05:35:21.805718Z","iopub.status.idle":"2022-09-27T05:35:21.814268Z","shell.execute_reply.started":"2022-09-27T05:35:21.805683Z","shell.execute_reply":"2022-09-27T05:35:21.813125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p ~/.cache/torch/hub/checkpoints\n!cp ../input/k/deepanshu0009/pyt-mayo-strip-ai-eda-train-efficientnet/model.pth \\\n    ~/.cache/torch/hub/checkpoints","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_efficientnet_b4', pretrained=True)\nmodel = torch.jit.load('../input/k/deepanshu0009/pyt-mayo-strip-ai-eda-train-efficientnet/model.pth')\nbatch_size = 1\ntest_loader = DataLoader(\n    ImgDataset(test_df), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:36:23.611502Z","iopub.execute_input":"2022-09-27T05:36:23.611927Z","iopub.status.idle":"2022-09-27T05:36:27.007194Z","shell.execute_reply.started":"2022-09-27T05:36:23.611886Z","shell.execute_reply":"2022-09-27T05:36:27.006127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anss, ids = predict(model, test_loader)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:36:58.253396Z","iopub.execute_input":"2022-09-27T05:36:58.254459Z","iopub.status.idle":"2022-09-27T05:36:58.436166Z","shell.execute_reply.started":"2022-09-27T05:36:58.254411Z","shell.execute_reply":"2022-09-27T05:36:58.434994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prob = pd.DataFrame({\"CE\" : anss[:,0], \"LAA\" : anss[:,1], \"id\" : ids}).groupby(\"id\").mean()","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:37:23.023965Z","iopub.execute_input":"2022-09-27T05:37:23.024401Z","iopub.status.idle":"2022-09-27T05:37:23.040679Z","shell.execute_reply.started":"2022-09-27T05:37:23.024357Z","shell.execute_reply":"2022-09-27T05:37:23.039582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:37:38.948065Z","iopub.execute_input":"2022-09-27T05:37:38.948465Z","iopub.status.idle":"2022-09-27T05:37:38.958617Z","shell.execute_reply.started":"2022-09-27T05:37:38.94843Z","shell.execute_reply":"2022-09-27T05:37:38.957607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.CE = prob.CE.to_list()\nsubmission.LAA = prob.LAA.to_list()","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:37:53.341002Z","iopub.execute_input":"2022-09-27T05:37:53.341406Z","iopub.status.idle":"2022-09-27T05:37:53.347511Z","shell.execute_reply.started":"2022-09-27T05:37:53.341372Z","shell.execute_reply":"2022-09-27T05:37:53.346492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:38:00.51046Z","iopub.execute_input":"2022-09-27T05:38:00.510978Z","iopub.status.idle":"2022-09-27T05:38:00.540162Z","shell.execute_reply.started":"2022-09-27T05:38:00.510931Z","shell.execute_reply":"2022-09-27T05:38:00.538906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T05:38:09.16677Z","iopub.execute_input":"2022-09-27T05:38:09.167188Z","iopub.status.idle":"2022-09-27T05:38:09.176714Z","shell.execute_reply.started":"2022-09-27T05:38:09.167155Z","shell.execute_reply":"2022-09-27T05:38:09.175708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}