{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In the below notebook I show a way doing ensemble between solutions and use the new Kaggle env. feature when submitting, all in all this saves both memory when running multiple solutions and also your valuable weekly GPU quota.\n \nCredit to the public solution used:\n\n* https://www.kaggle.com/code/thedevastator/train-infer-coatnet-efficientnet\n* https://www.kaggle.com/code/rishavnandi/mayo-clinic\n* https://www.kaggle.com/code/realneuralnetwork/cnn-strip-ai-inference\n\nHow it works: We save the codes for every solution in an own python file which we then run separately isolated memory. We also take advantages of the new Kaggle env. feature that checks if we are in the hidden test running phase or only in the submitting phase, and use this by saving a submission sample in submitting phase and when in the phase of running the hidden private test phase, using the real code. This use only seconds of the GPU quota.","metadata":{}},{"cell_type":"code","source":"%%writefile sub5v1.py\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:01:29.619297Z\",\"iopub.execute_input\":\"2022-08-14T13:01:29.620204Z\",\"iopub.status.idle\":\"2022-08-14T13:01:33.146748Z\",\"shell.execute_reply.started\":\"2022-08-14T13:01:29.620095Z\",\"shell.execute_reply\":\"2022-08-14T13:01:33.145603Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nimport os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nimport torch\nfrom torch import nn\nimport seaborn as sns\nfrom torchvision import models\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.notebook import tqdm\nfrom torch.optim import lr_scheduler\nimport warnings\nwarnings.filterwarnings(\"ignore\")\ngc.enable()\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:01:33.149288Z\",\"iopub.execute_input\":\"2022-08-14T13:01:33.150422Z\",\"iopub.status.idle\":\"2022-08-14T13:01:33.188547Z\",\"shell.execute_reply.started\":\"2022-08-14T13:01:33.150358Z\",\"shell.execute_reply\":\"2022-08-14T13:01:33.187583Z\"},\"jupyter\":{\"outputs_hidden\":false}}\ndebug = False\ngenerate_new = True\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\ntest_df\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:01:33.190002Z\",\"iopub.execute_input\":\"2022-08-14T13:01:33.190474Z\",\"iopub.status.idle\":\"2022-08-14T13:02:10.521775Z\",\"shell.execute_reply.started\":\"2022-08-14T13:01:33.19044Z\",\"shell.execute_reply\":\"2022-08-14T13:02:10.520703Z\"},\"jupyter\":{\"outputs_hidden\":false}}\ntry:\n    os.mkdir(\"../test/\")\nexcept:\n    pass\nfor i in tqdm(range(test_df.shape[0])):\n    img_id = test_df.iloc[i].image_id\n    try:\n        sz = os.path.getsize(dirs[1] + img_id + \".tif\")\n    except:\n        sz = 1000000000\n    if(sz > 8e8):\n        img = np.zeros((512,512,3), np.uint8)\n    else:\n        try:\n            img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n        except:\n            img = np.zeros((512,512,3), np.uint8)\n    cv2.imwrite(f\"../test/{img_id}.jpg\", img)\n    del img\n    gc.collect()\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:02:10.524379Z\",\"iopub.execute_input\":\"2022-08-14T13:02:10.525317Z\",\"iopub.status.idle\":\"2022-08-14T13:02:10.535293Z\",\"shell.execute_reply.started\":\"2022-08-14T13:02:10.52528Z\",\"shell.execute_reply\":\"2022-08-14T13:02:10.534318Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nclass ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"../test/\", \"../train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        try:\n            image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        except:\n            image = np.zeros((512,512,3), np.uint8)\n        label = 0\n        try:\n            if len(image.shape) == 5:\n                image = image.squeeze().transpose(1, 2, 0)\n            image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        except:\n            image = np.zeros((3, 512, 512))\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        patient_id = self.df.iloc[index].patient_id\n        return image, label, patient_id\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:02:10.536682Z\",\"iopub.execute_input\":\"2022-08-14T13:02:10.537213Z\",\"iopub.status.idle\":\"2022-08-14T13:02:10.549895Z\",\"shell.execute_reply.started\":\"2022-08-14T13:02:10.53718Z\",\"shell.execute_reply\":\"2022-08-14T13:02:10.548781Z\"},\"jupyter\":{\"outputs_hidden\":false}}\ndef predict(model, dataloader):\n    model.cuda()\n    model.eval()\n    dataloader = dataloader\n    outputs = []\n    s = nn.Softmax(dim=1)\n    ids = []\n    for item in tqdm(dataloader, leave=False):\n        patient_id = item[2][0]\n        try:\n            images = item[0].cuda().float()\n            ids.append(patient_id)\n            output = model(images)\n            outputs.append(s(output.cpu()[:,:2])[0].detach().numpy())\n        except:\n            ids.append(patient_id)\n            outputs.append(s(torch.tensor([[1, 1]]).float())[0].detach().numpy())\n    return np.array(outputs), ids\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:02:10.551415Z\",\"iopub.execute_input\":\"2022-08-14T13:02:10.551793Z\",\"iopub.status.idle\":\"2022-08-14T13:02:11.793961Z\",\"shell.execute_reply.started\":\"2022-08-14T13:02:10.551759Z\",\"shell.execute_reply\":\"2022-08-14T13:02:11.792952Z\"},\"jupyter\":{\"outputs_hidden\":false}}\n#model = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_efficientnet_b4', pretrained=True)\nmodel = torch.jit.load('../input/cnnstripai/model.pth')\nbatch_size = 1\ntest_loader = DataLoader(\n    ImgDataset(test_df), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:02:11.79537Z\",\"iopub.execute_input\":\"2022-08-14T13:02:11.795739Z\",\"iopub.status.idle\":\"2022-08-14T13:02:21.911804Z\",\"shell.execute_reply.started\":\"2022-08-14T13:02:11.795706Z\",\"shell.execute_reply\":\"2022-08-14T13:02:21.910591Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nanss, ids = predict(model, test_loader)\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:02:21.913618Z\",\"iopub.execute_input\":\"2022-08-14T13:02:21.914803Z\",\"iopub.status.idle\":\"2022-08-14T13:02:21.930366Z\",\"shell.execute_reply.started\":\"2022-08-14T13:02:21.914761Z\",\"shell.execute_reply\":\"2022-08-14T13:02:21.929157Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nprob = pd.DataFrame({\"CE\" : anss[:,0], \"LAA\" : anss[:,1], \"id\" : ids}).groupby(\"id\").mean()\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:02:21.93168Z\",\"iopub.execute_input\":\"2022-08-14T13:02:21.932694Z\",\"iopub.status.idle\":\"2022-08-14T13:02:21.952434Z\",\"shell.execute_reply.started\":\"2022-08-14T13:02:21.932659Z\",\"shell.execute_reply\":\"2022-08-14T13:02:21.951583Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nsubmission = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:02:21.955927Z\",\"iopub.execute_input\":\"2022-08-14T13:02:21.956195Z\",\"iopub.status.idle\":\"2022-08-14T13:02:21.961367Z\",\"shell.execute_reply.started\":\"2022-08-14T13:02:21.956171Z\",\"shell.execute_reply\":\"2022-08-14T13:02:21.960361Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nsubmission.CE = prob.CE.to_list()\nsubmission.LAA = prob.LAA.to_list()\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:02:21.963153Z\",\"iopub.execute_input\":\"2022-08-14T13:02:21.964035Z\",\"iopub.status.idle\":\"2022-08-14T13:02:21.97688Z\",\"shell.execute_reply.started\":\"2022-08-14T13:02:21.963995Z\",\"shell.execute_reply\":\"2022-08-14T13:02:21.975892Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nsubmission\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-14T13:02:21.97819Z\",\"iopub.execute_input\":\"2022-08-14T13:02:21.979233Z\",\"iopub.status.idle\":\"2022-08-14T13:02:21.988559Z\",\"shell.execute_reply.started\":\"2022-08-14T13:02:21.9792Z\",\"shell.execute_reply\":\"2022-08-14T13:02:21.987107Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nsubmission.to_csv(\"sub1.csv\", index = False)\n\n# %% [code] {\"jupyter\":{\"outputs_hidden\":false}}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile sub5v2.py\n\n# %% [code] {\"papermill\":{\"duration\":3.063995,\"end_time\":\"2022-07-08T14:24:41.045696\",\"exception\":false,\"start_time\":\"2022-07-08T14:24:37.981701\",\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:11.14947Z\",\"iopub.execute_input\":\"2022-07-13T09:14:11.150133Z\",\"iopub.status.idle\":\"2022-07-13T09:14:11.157336Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:11.150096Z\",\"shell.execute_reply\":\"2022-07-13T09:14:11.156361Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nimport os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nimport torch\nfrom torch import nn\nimport seaborn as sns\nfrom torchvision import models\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.notebook import tqdm\nfrom torch.optim import lr_scheduler\nimport warnings\nwarnings.filterwarnings(\"ignore\")\ngc.enable()\n\n# %% [code] {\"papermill\":{\"duration\":0.02504,\"end_time\":\"2022-07-08T14:24:41.073811\",\"exception\":false,\"start_time\":\"2022-07-08T14:24:41.048771\",\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:11.256259Z\",\"iopub.execute_input\":\"2022-07-13T09:14:11.256823Z\",\"iopub.status.idle\":\"2022-07-13T09:14:11.271188Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:11.256784Z\",\"shell.execute_reply\":\"2022-07-13T09:14:11.270248Z\"},\"jupyter\":{\"outputs_hidden\":false}}\ndebug = False\ngenerate_new = True\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\ntest_df\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:11.403684Z\",\"iopub.execute_input\":\"2022-07-13T09:14:11.404538Z\",\"iopub.status.idle\":\"2022-07-13T09:14:11.410775Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:11.404503Z\",\"shell.execute_reply\":\"2022-07-13T09:14:11.40947Z\"},\"jupyter\":{\"outputs_hidden\":false}}\n#test_df = pd.DataFrame({\"image_id\" : [\"006388_0\", \"008e5c_0\", \"00c058_0\", \"01adc5_0\", \"01adc5_0\"], \"patient_id\" : [\"006388\", \"008e5c\", \"00c058\", \"01adc5\", \"01adc5\"]})\n\n# %% [code] {\"papermill\":{\"duration\":69.477711,\"end_time\":\"2022-07-08T14:25:50.554416\",\"exception\":false,\"start_time\":\"2022-07-08T14:24:41.076705\",\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:11.539492Z\",\"iopub.execute_input\":\"2022-07-13T09:14:11.540398Z\",\"iopub.status.idle\":\"2022-07-13T09:14:56.867349Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:11.540365Z\",\"shell.execute_reply\":\"2022-07-13T09:14:56.866398Z\"},\"jupyter\":{\"outputs_hidden\":false}}\ntry:\n    os.mkdir(\"../test/\")\nexcept:\n    pass\nfor i in tqdm(range(test_df.shape[0])):\n    img_id = test_df.iloc[i].image_id\n    try:\n        sz = os.path.getsize(dirs[1] + img_id + \".tif\")\n    except:\n        sz = 1000000000\n    if(sz > 8e8):\n        img = np.zeros((512,512,3), np.uint8)\n    else:\n        try:\n            img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n        except:\n            img = np.zeros((512,512,3), np.uint8)\n    cv2.imwrite(f\"../test/{img_id}.jpg\", img)\n    del img\n    gc.collect()\n\n# %% [code] {\"papermill\":{\"duration\":null,\"end_time\":null,\"exception\":null,\"start_time\":null,\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:56.869144Z\",\"iopub.execute_input\":\"2022-07-13T09:14:56.870058Z\",\"iopub.status.idle\":\"2022-07-13T09:14:56.881361Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:56.87002Z\",\"shell.execute_reply\":\"2022-07-13T09:14:56.880454Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nclass ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"../test/\", \"../train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        try:\n            image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        except:\n            image = np.zeros((512,512,3), np.uint8)\n        label = 0\n        try:\n            if len(image.shape) == 5:\n                image = image.squeeze().transpose(1, 2, 0)\n            image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        except:\n            image = np.zeros((3, 512, 512))\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        patient_id = self.df.iloc[index].patient_id\n        return image, label, patient_id\n\n# %% [code] {\"papermill\":{\"duration\":null,\"end_time\":null,\"exception\":null,\"start_time\":null,\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:56.882647Z\",\"iopub.execute_input\":\"2022-07-13T09:14:56.882989Z\",\"iopub.status.idle\":\"2022-07-13T09:14:56.89465Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:56.882956Z\",\"shell.execute_reply\":\"2022-07-13T09:14:56.893659Z\"},\"jupyter\":{\"outputs_hidden\":false}}\ndef predict(model, dataloader):\n    model.cuda()\n    model.eval()\n    dataloader = dataloader\n    outputs = []\n    s = nn.Softmax(dim=1)\n    ids = []\n    for item in tqdm(dataloader, leave=False):\n        patient_id = item[2][0]\n        try:\n            images = item[0].cuda().float()\n            ids.append(patient_id)\n            output = model(images)\n            outputs.append(s(output.cpu()[:,:2])[0].detach().numpy())\n        except:\n            ids.append(patient_id)\n            outputs.append(s(torch.tensor([[1, 1]]).float())[0].detach().numpy())\n    return np.array(outputs), ids\n\n# %% [code] {\"papermill\":{\"duration\":null,\"end_time\":null,\"exception\":null,\"start_time\":null,\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:56.898742Z\",\"iopub.execute_input\":\"2022-07-13T09:14:56.899357Z\",\"iopub.status.idle\":\"2022-07-13T09:14:57.620651Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:56.899332Z\",\"shell.execute_reply\":\"2022-07-13T09:14:57.619672Z\"},\"jupyter\":{\"outputs_hidden\":false}}\n#model = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_efficientnet_b4', pretrained=True)\nmodel = torch.jit.load('../input/sub5v2v5/model.pth')\nbatch_size = 1\ntest_loader = DataLoader(\n    ImgDataset(test_df), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\n\n# %% [code] {\"papermill\":{\"duration\":null,\"end_time\":null,\"exception\":null,\"start_time\":null,\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:57.622967Z\",\"iopub.execute_input\":\"2022-07-13T09:14:57.623542Z\",\"iopub.status.idle\":\"2022-07-13T09:14:59.054783Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:57.623504Z\",\"shell.execute_reply\":\"2022-07-13T09:14:59.053568Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nanss, ids = predict(model, test_loader)\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:59.070575Z\",\"iopub.execute_input\":\"2022-07-13T09:14:59.070984Z\",\"iopub.status.idle\":\"2022-07-13T09:14:59.081505Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:59.070945Z\",\"shell.execute_reply\":\"2022-07-13T09:14:59.080336Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nprob = pd.DataFrame({\"CE\" : anss[:,0], \"LAA\" : anss[:,1], \"id\" : ids}).groupby(\"id\").mean()\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:59.083142Z\",\"iopub.execute_input\":\"2022-07-13T09:14:59.083588Z\",\"iopub.status.idle\":\"2022-07-13T09:14:59.092109Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:59.083551Z\",\"shell.execute_reply\":\"2022-07-13T09:14:59.090657Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nsubmission = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:59.094307Z\",\"iopub.execute_input\":\"2022-07-13T09:14:59.094793Z\",\"iopub.status.idle\":\"2022-07-13T09:14:59.101729Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:59.094757Z\",\"shell.execute_reply\":\"2022-07-13T09:14:59.100743Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nsubmission.CE = prob.CE.to_list()\nsubmission.LAA = prob.LAA.to_list()\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:59.10537Z\",\"iopub.execute_input\":\"2022-07-13T09:14:59.105812Z\",\"iopub.status.idle\":\"2022-07-13T09:14:59.119463Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:59.10578Z\",\"shell.execute_reply\":\"2022-07-13T09:14:59.1185Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nsubmission\n\n# %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-07-13T09:14:59.120955Z\",\"iopub.execute_input\":\"2022-07-13T09:14:59.121363Z\",\"iopub.status.idle\":\"2022-07-13T09:14:59.129202Z\",\"shell.execute_reply.started\":\"2022-07-13T09:14:59.121327Z\",\"shell.execute_reply\":\"2022-07-13T09:14:59.128302Z\"},\"jupyter\":{\"outputs_hidden\":false}}\nsubmission.to_csv(\"sub2.csv\", index = False)\n\n# %% [code] {\"jupyter\":{\"outputs_hidden\":false}}\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile sub5v3.py\n# %% [markdown]\n# ### Install Required Libraries\n\n# %% [code]\nimport sys\nsys.path.append('../input/einops')\n\n# %% [code]\nsys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')\nfrom efficientnet_pytorch import EfficientNet\n# !pip install --upgrade efficientnet-pytorch\n\n# %% [code]\n# !pip install einops\n\n# %% [markdown]\n# ### Import Required Libraries\n\n# %% [code] {\"papermill\":{\"duration\":3.063995,\"end_time\":\"2022-07-08T14:24:41.045696\",\"exception\":false,\"start_time\":\"2022-07-08T14:24:37.981701\",\"status\":\"completed\"},\"tags\":[]}\nimport os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nimport torch\nimport torch.nn as nn\nfrom einops import rearrange\nfrom einops.layers.torch import Rearrange\nimport seaborn as sns\nfrom torchvision import models\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nfrom efficientnet_pytorch import EfficientNet\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.notebook import tqdm\nfrom torch.optim import lr_scheduler\nfrom random import randint\nimport warnings\nwarnings.filterwarnings(\"ignore\")\ngc.enable()\n\n# %% [markdown]\n# ### Set Seed for Reproducibility\n\n# %% [code]\ndef seed_everything(seed_value):\n    random.seed(seed_value)\n    np.random.seed(seed_value)\n    torch.manual_seed(seed_value)\n    os.environ['PYTHONHASHSEED'] = str(seed_value)\n    \n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = True\n\nseed = 42\nseed_everything(seed)\n\n# %% [markdown]\n# ### Read the Data \n\n# %% [code] {\"papermill\":{\"duration\":0.02504,\"end_time\":\"2022-07-08T14:24:41.073811\",\"exception\":false,\"start_time\":\"2022-07-08T14:24:41.048771\",\"status\":\"completed\"},\"tags\":[]}\ndebug = False\ngenerate_new = False\ntrain_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/train.csv\").head(10 if debug else 1000)\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\n\n# %% [code]\nsns.countplot(train_df.label)\n\n# %% [markdown]\n# ### Balance the classes\n\n# %% [code]\nmax_count = max(train_df.label.value_counts())\nfor label in train_df.label.unique():\n    df = train_df.loc[train_df.label == label]\n    while(train_df.label.value_counts()[label] < max_count):\n        train_df = pd.concat([train_df, df.head(max_count - train_df.label.value_counts()[label])], axis = 0)\n\n# %% [code]\nsns.countplot(train_df.label)\n\n# %% [code] {\"papermill\":{\"duration\":69.477711,\"end_time\":\"2022-07-08T14:25:50.554416\",\"exception\":false,\"start_time\":\"2022-07-08T14:24:41.076705\",\"status\":\"completed\"},\"tags\":[]}\nif(generate_new):\n    os.mkdir(\"./train/\")\n    os.mkdir(\"./test/\")\n    for i in tqdm(range(test_df.shape[0])):\n        img_id = test_df.iloc[i].image_id\n        img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n        cv2.imwrite(f\"./test/{img_id}.jpg\", img)\n        del img\n        gc.collect()\n    for i in tqdm(range(train_df.shape[0])):\n        img_id = train_df.iloc[i].image_id\n        img = cv2.resize(tifffile.imread(dirs[0] + img_id + \".tif\"), (512, 512))\n        cv2.imwrite(f\"./train/{img_id}.jpg\", img)\n        del img\n        gc.collect()\n\n# %% [markdown]\n# ### Dataset Class\n\n# %% [code] {\"papermill\":{\"duration\":null,\"end_time\":null,\"exception\":null,\"start_time\":null,\"status\":\"completed\"},\"tags\":[]}\nclass ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"./test/\", \"./train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        if len(image.shape) == 5:\n            image = image.squeeze().transpose(1, 2, 0)\n        image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        label = None\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        return image, label\n\n# %% [markdown]\n# ### Train model\n\n# %% [code] {\"papermill\":{\"duration\":null,\"end_time\":null,\"exception\":null,\"start_time\":null,\"status\":\"completed\"},\"tags\":[]}\ndef train_model(model, dataloaders_dict, criterion, optimizer, num_epochs):\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        model.cuda()\n        \n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n                \n            epoch_loss = 0.0\n            epoch_acc = 0\n            \n            dataloader = dataloaders_dict[phase]\n            for item in tqdm(dataloader, leave=False):\n                images = item[0].cuda().float()\n                classes = item[1].cuda().long()\n\n                optimizer.zero_grad()\n                \n                with torch.set_grad_enabled(phase == 'train'):\n                    output = model(images)\n                    loss = criterion(output, classes)\n                    _, preds = torch.max(output, 1)\n\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                    epoch_loss += loss.item() * len(output)\n                    epoch_acc += torch.sum(preds == classes.data)\n                    \n\n            data_size = len(dataloader.dataset)\n            epoch_loss = epoch_loss / data_size\n            epoch_acc = epoch_acc.double() / data_size\n\n            print(f'Epoch {epoch + 1}/{num_epochs} | {phase:^5} | Loss: {epoch_loss:.4f} | Acc: {epoch_acc:.4f}')\n        \n        if epoch_acc > best_acc:\n            traced = torch.jit.trace(model.cpu(), torch.rand(1, 3, 512, 512))\n            traced.save('model.pth')\n            best_acc = epoch_acc\n\n# %% [code]\ndef conv_3x3_bn(inp, oup, image_size, downsample=False):\n    stride = 1 if downsample == False else 2\n    return nn.Sequential(\n        nn.Conv2d(inp, oup, 3, stride, 1, bias=False),\n        nn.BatchNorm2d(oup),\n        nn.GELU()\n    )\n\n\nclass PreNorm(nn.Module):\n    def __init__(self, dim, fn, norm):\n        super().__init__()\n        self.norm = norm(dim)\n        self.fn = fn\n\n    def forward(self, x, **kwargs):\n        return self.fn(self.norm(x), **kwargs)\n\n\nclass SE(nn.Module):\n    def __init__(self, inp, oup, expansion=0.25):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Sequential(\n            nn.Linear(oup, int(inp * expansion), bias=False),\n            nn.GELU(),\n            nn.Linear(int(inp * expansion), oup, bias=False),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        b, c, _, _ = x.size()\n        y = self.avg_pool(x).view(b, c)\n        y = self.fc(y).view(b, c, 1, 1)\n        return x * y\n\n\nclass FeedForward(nn.Module):\n    def __init__(self, dim, hidden_dim, dropout=0.):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, dim),\n            nn.Dropout(dropout)\n        )\n\n    def forward(self, x):\n        return self.net(x)\n\n\nclass MBConv(nn.Module):\n    def __init__(self, inp, oup, image_size, downsample=False, expansion=4):\n        super().__init__()\n        self.downsample = downsample\n        stride = 1 if self.downsample == False else 2\n        hidden_dim = int(inp * expansion)\n\n        if self.downsample:\n            self.pool = nn.MaxPool2d(3, 2, 1)\n            self.proj = nn.Conv2d(inp, oup, 1, 1, 0, bias=False)\n\n        if expansion == 1:\n            self.conv = nn.Sequential(\n                # dw\n                nn.Conv2d(hidden_dim, hidden_dim, 3, stride,\n                          1, groups=hidden_dim, bias=False),\n                nn.BatchNorm2d(hidden_dim),\n                nn.GELU(),\n                # pw-linear\n                nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n                nn.BatchNorm2d(oup),\n            )\n        else:\n            self.conv = nn.Sequential(\n                # pw\n                # down-sample in the first conv\n                nn.Conv2d(inp, hidden_dim, 1, stride, 0, bias=False),\n                nn.BatchNorm2d(hidden_dim),\n                nn.GELU(),\n                # dw\n                nn.Conv2d(hidden_dim, hidden_dim, 3, 1, 1,\n                          groups=hidden_dim, bias=False),\n                nn.BatchNorm2d(hidden_dim),\n                nn.GELU(),\n                SE(inp, hidden_dim),\n                # pw-linear\n                nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n                nn.BatchNorm2d(oup),\n            )\n        \n        self.conv = PreNorm(inp, self.conv, nn.BatchNorm2d)\n\n    def forward(self, x):\n        if self.downsample:\n            return self.proj(self.pool(x)) + self.conv(x)\n        else:\n            return x + self.conv(x)\n\n\nclass Attention(nn.Module):\n    def __init__(self, inp, oup, image_size, heads=8, dim_head=32, dropout=0.):\n        super().__init__()\n        inner_dim = dim_head * heads\n        project_out = not (heads == 1 and dim_head == inp)\n\n        self.ih, self.iw = image_size\n\n        self.heads = heads\n        self.scale = dim_head ** -0.5\n\n        # parameter table of relative position bias\n        self.relative_bias_table = nn.Parameter(\n            torch.zeros((2 * self.ih - 1) * (2 * self.iw - 1), heads))\n\n        coords = torch.meshgrid((torch.arange(self.ih), torch.arange(self.iw)))\n        coords = torch.flatten(torch.stack(coords), 1)\n        relative_coords = coords[:, :, None] - coords[:, None, :]\n\n        relative_coords[0] += self.ih - 1\n        relative_coords[1] += self.iw - 1\n        relative_coords[0] *= 2 * self.iw - 1\n        relative_coords = rearrange(relative_coords, 'c h w -> h w c')\n        relative_index = relative_coords.sum(-1).flatten().unsqueeze(1)\n        self.register_buffer(\"relative_index\", relative_index)\n\n        self.attend = nn.Softmax(dim=-1)\n        self.to_qkv = nn.Linear(inp, inner_dim * 3, bias=False)\n\n        self.to_out = nn.Sequential(\n            nn.Linear(inner_dim, oup),\n            nn.Dropout(dropout)\n        ) if project_out else nn.Identity()\n\n    def forward(self, x):\n        qkv = self.to_qkv(x).chunk(3, dim=-1)\n        q, k, v = map(lambda t: rearrange(\n            t, 'b n (h d) -> b h n d', h=self.heads), qkv)\n\n        dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale\n\n        # Use \"gather\" for more efficiency on GPUs\n        relative_bias = self.relative_bias_table.gather(\n            0, self.relative_index.repeat(1, self.heads))\n        relative_bias = rearrange(\n            relative_bias, '(h w) c -> 1 c h w', h=self.ih*self.iw, w=self.ih*self.iw)\n        dots = dots + relative_bias\n\n        attn = self.attend(dots)\n        out = torch.matmul(attn, v)\n        out = rearrange(out, 'b h n d -> b n (h d)')\n        out = self.to_out(out)\n        return out\n\n\nclass Transformer(nn.Module):\n    def __init__(self, inp, oup, image_size, heads=8, dim_head=32, downsample=False, dropout=0.):\n        super().__init__()\n        hidden_dim = int(inp * 4)\n\n        self.ih, self.iw = image_size\n        self.downsample = downsample\n\n        if self.downsample:\n            self.pool1 = nn.MaxPool2d(3, 2, 1)\n            self.pool2 = nn.MaxPool2d(3, 2, 1)\n            self.proj = nn.Conv2d(inp, oup, 1, 1, 0, bias=False)\n\n        self.attn = Attention(inp, oup, image_size, heads, dim_head, dropout)\n        self.ff = FeedForward(oup, hidden_dim, dropout)\n\n        self.attn = nn.Sequential(\n            Rearrange('b c ih iw -> b (ih iw) c'),\n            PreNorm(inp, self.attn, nn.LayerNorm),\n            Rearrange('b (ih iw) c -> b c ih iw', ih=self.ih, iw=self.iw)\n        )\n\n        self.ff = nn.Sequential(\n            Rearrange('b c ih iw -> b (ih iw) c'),\n            PreNorm(oup, self.ff, nn.LayerNorm),\n            Rearrange('b (ih iw) c -> b c ih iw', ih=self.ih, iw=self.iw)\n        )\n\n    def forward(self, x):\n        if self.downsample:\n            x = self.proj(self.pool1(x)) + self.attn(self.pool2(x))\n        else:\n            x = x + self.attn(x)\n        x = x + self.ff(x)\n        return x\n\n\nclass CoAtNet(nn.Module):\n    def __init__(self, image_size, in_channels, num_blocks, channels, num_classes=1000, block_types=['C', 'C', 'T', 'T']):\n        super().__init__()\n        ih, iw = image_size\n        block = {'C': MBConv, 'T': Transformer}\n\n        self.s0 = self._make_layer(\n            conv_3x3_bn, in_channels, channels[0], num_blocks[0], (ih // 2, iw // 2))\n        self.s1 = self._make_layer(\n            block[block_types[0]], channels[0], channels[1], num_blocks[1], (ih // 4, iw // 4))\n        self.s2 = self._make_layer(\n            block[block_types[1]], channels[1], channels[2], num_blocks[2], (ih // 8, iw // 8))\n        self.s3 = self._make_layer(\n            block[block_types[2]], channels[2], channels[3], num_blocks[3], (ih // 16, iw // 16))\n        self.s4 = self._make_layer(\n            block[block_types[3]], channels[3], channels[4], num_blocks[4], (ih // 32, iw // 32))\n\n        self.pool = nn.AvgPool2d(ih // 32, 1)\n        self.fc = nn.Linear(channels[-1], num_classes, bias=False)\n\n    def forward(self, x):\n        x = self.s0(x)\n        x = self.s1(x)\n        x = self.s2(x)\n        x = self.s3(x)\n        x = self.s4(x)\n\n        x = self.pool(x).view(-1, x.shape[1])\n        x = self.fc(x)\n        return x\n\n    def _make_layer(self, block, inp, oup, depth, image_size):\n        layers = nn.ModuleList([])\n        for i in range(depth):\n            if i == 0:\n                layers.append(block(inp, oup, image_size, downsample=True))\n            else:\n                layers.append(block(oup, oup, image_size))\n        return nn.Sequential(*layers)\n\n# %% [code] {\"papermill\":{\"duration\":null,\"end_time\":null,\"exception\":null,\"start_time\":null,\"status\":\"completed\"},\"tags\":[]}\n#model = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_efficientnet_b4', pretrained=True)\n#model.fc = nn.Linear(512, 2)\nnum_blocks = [2, 2, 12, 28, 2]\nchannels = [64, 64, 128, 256, 512]\nmodel = CoAtNet((512, 512), 3, num_blocks, channels, num_classes=2)\ntrain, val = train_test_split(train_df, test_size=0.2, random_state=42, stratify = train_df.label)\nbatch_size = 1\ntrain_loader = DataLoader(\n    ImgDataset(train), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\nval_loader = DataLoader(\n    ImgDataset(val), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\ndataloaders_dict = {\"train\": train_loader, \"val\": val_loader}\ncriterion = nn.CrossEntropyLoss()\n\n# %% [markdown]\n# ### Run Training\n\n# %% [code] {\"papermill\":{\"duration\":null,\"end_time\":null,\"exception\":null,\"start_time\":null,\"status\":\"completed\"},\"tags\":[]}\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\ntrain_model(model, dataloaders_dict, criterion, optimizer, 1)\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)\ntrain_model(model, dataloaders_dict, criterion, optimizer, 1)\n\n# %% [code]\n\n\nimport os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nimport torch\nfrom torch import nn\nimport seaborn as sns\nfrom torchvision import models\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.notebook import tqdm\nfrom torch.optim import lr_scheduler\nimport warnings\nwarnings.filterwarnings(\"ignore\")\ngc.enable()\n\n\n\ndebug = False\ngenerate_new = True\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\nif(test_df.shape[0] == 4):\n    test_df = pd.concat([test_df for i in range(25)])\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\ntest_df\n\n\n\ntry:\n    os.mkdir(\"../test/\")\nexcept:\n    pass\nfor i in tqdm(range(test_df.shape[0])):\n    img_id = test_df.iloc[i].image_id\n    try:\n        sz = os.path.getsize(dirs[1] + img_id + \".tif\")\n    except:\n        sz = 1000000000\n    if(sz > 8e8):\n        img = np.zeros((512,512,3), np.uint8)\n    else:\n        try:\n            img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n        except:\n            img = np.zeros((512,512,3), np.uint8)\n    cv2.imwrite(f\"../test/{img_id}.jpg\", img)\n    del img\n    gc.collect()\n    \n    \n    \nclass ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"../test/\", \"../train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        try:\n            image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        except:\n            image = np.zeros((512,512,3), np.uint8)\n        label = 0\n        try:\n            if len(image.shape) == 5:\n                image = image.squeeze().transpose(1, 2, 0)\n            image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        except:\n            image = np.zeros((3, 512, 512))\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        patient_id = self.df.iloc[index].patient_id\n        return image, label, patient_id\n\n        \n        \ndef predict(model, dataloader):\n    model.cuda()\n    model.eval()\n    dataloader = dataloader\n    outputs = []\n    s = nn.Softmax(dim=1)\n    ids = []\n    for item in tqdm(dataloader, leave=False):\n        patient_id = item[2][0]\n        try:\n            images = item[0].cuda().float()\n            ids.append(patient_id)\n            output = model(images)\n            outputs.append(s(output.cpu()[:,:2])[0].detach().numpy())\n        except:\n            ids.append(patient_id)\n            outputs.append(s(torch.tensor([[1, 1]]).float())[0].detach().numpy())\n    return np.array(outputs), ids\n    \n    \n    \nmodel = torch.jit.load('model.pth')\nbatch_size = 1\ntest_loader = DataLoader(\n    ImgDataset(test_df), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\n\nanss, ids = predict(model, test_loader)\n\nprob = pd.DataFrame({\"CE\" : anss[:,0], \"LAA\" : anss[:,1], \"id\" : ids}).groupby(\"id\").mean()\n\nsubmission = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")\n\nsubmission.CE = prob.CE.to_list()\nsubmission.LAA = prob.LAA.to_list()\n\nsubmission.to_csv(\"sub3.csv\", index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    \n    !python sub5v3.py\n    !python sub5v1.py\n    !python sub5v2.py\n    \nelse:\n    \n    import pandas as pd\n    submission_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")\n    submission_df.to_csv(\"sub1.csv\", index=False)\n    submission_df.to_csv(\"sub2.csv\", index=False)\n    submission_df.to_csv(\"sub3.csv\", index=False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nsub1 = pd.read_csv('sub1.csv')\nsub2 = pd.read_csv('sub2.csv')\nsub3 = pd.read_csv('sub3.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_final = sub1.copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_final['CE'] = sub1.CE*1/3 + sub2.CE*1/3 + sub3.CE*1/3\nsub_final['LAA'] = sub1.LAA*1/3 + sub2.LAA*1/3 + sub3.CE*1/3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_final","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_final.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}