{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nimport torch\nfrom torch import nn\nimport seaborn as sns\nfrom torchvision import models\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.notebook import tqdm\nfrom torch.optim import lr_scheduler\nimport warnings\nwarnings.filterwarnings(\"ignore\")\ngc.enable()","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":3.063995,"end_time":"2022-07-08T14:24:41.045696","exception":false,"start_time":"2022-07-08T14:24:37.981701","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-27T14:10:31.812857Z","iopub.execute_input":"2022-07-27T14:10:31.813272Z","iopub.status.idle":"2022-07-27T14:10:34.300221Z","shell.execute_reply.started":"2022-07-27T14:10:31.813229Z","shell.execute_reply":"2022-07-27T14:10:34.299004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train ","metadata":{}},{"cell_type":"code","source":"debug = True\ngenerate_new = True\ntrain_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/train.csv\").head(10 if debug else 754)\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T14:10:41.32655Z","iopub.execute_input":"2022-07-27T14:10:41.327596Z","iopub.status.idle":"2022-07-27T14:10:41.3415Z","shell.execute_reply.started":"2022-07-27T14:10:41.327529Z","shell.execute_reply":"2022-07-27T14:10:41.340092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if(generate_new):\n    os.mkdir(\"./train/\")\n    os.mkdir(\"./test/\")\n    for i in tqdm(range(test_df.shape[0])):\n        img_id = test_df.iloc[i].image_id\n        img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (1024, 1024))\n        cv2.imwrite(f\"./test/{img_id}.jpg\", img)\n        del img\n        gc.collect()\n    for i in tqdm(range(train_df.shape[0])):\n        img_id = train_df.iloc[i].image_id\n        img = cv2.resize(tifffile.imread(dirs[0] + img_id + \".tif\"), (1024, 1024))\n        cv2.imwrite(f\"./train/{img_id}.jpg\", img)\n        del img\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:14:12.611489Z","iopub.execute_input":"2022-07-24T16:14:12.611848Z","iopub.status.idle":"2022-07-24T16:18:06.204013Z","shell.execute_reply.started":"2022-07-24T16:14:12.611812Z","shell.execute_reply":"2022-07-24T16:18:06.203052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"./test/\", \"./train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        if len(image.shape) == 5:\n            image = image.squeeze().transpose(1, 2, 0)\n        image = cv2.resize(image, (1024, 1024)).transpose(2, 0, 1)\n        label = None\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:18:06.206528Z","iopub.execute_input":"2022-07-24T16:18:06.20743Z","iopub.status.idle":"2022-07-24T16:18:06.217303Z","shell.execute_reply.started":"2022-07-24T16:18:06.207391Z","shell.execute_reply":"2022-07-24T16:18:06.216312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs):\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        model.cuda()\n        \n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n                \n            epoch_loss = 0.0\n            epoch_acc = 0\n            \n            dataloader = dataloaders_dict[phase]\n            for item in tqdm(dataloader, leave=False):\n                images = item[0].cuda().float()\n                classes = item[1].cuda().long()\n\n                optimizer.zero_grad()\n                \n                with torch.set_grad_enabled(phase == 'train'):\n                    output = model(images)\n                    loss = criterion(output, classes)\n                    _, preds = torch.max(output, 1)\n\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                    epoch_loss += loss.item() * len(output)\n                    epoch_acc += torch.sum(preds == classes.data)\n                    \n\n            data_size = len(dataloader.dataset)\n            epoch_loss = epoch_loss / data_size\n            epoch_acc = epoch_acc.double() / data_size\n\n            print(f'Epoch {epoch + 1}/{num_epochs} | {phase:^5} | Loss: {epoch_loss:.4f} | Acc: {epoch_acc:.4f}')\n        \n        if epoch_acc > best_acc:\n            traced = torch.jit.trace(model.cpu(), torch.rand(1, 3, 1024, 1024))\n            traced.save('model.pth')\n            best_acc = epoch_acc","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:18:06.219919Z","iopub.execute_input":"2022-07-24T16:18:06.220285Z","iopub.status.idle":"2022-07-24T16:18:06.234112Z","shell.execute_reply.started":"2022-07-24T16:18:06.220234Z","shell.execute_reply":"2022-07-24T16:18:06.233054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_efficientnet_b4', pretrained=True)\ntrain, val = train_test_split(train_df, test_size=0.2, random_state=42, stratify = train_df.label)\nbatch_size = 1\ntrain_loader = DataLoader(\n    ImgDataset(train), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\nval_loader = DataLoader(\n    ImgDataset(val), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\ndataloaders_dict = {\"train\": train_loader, \"val\": val_loader}\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:18:06.236482Z","iopub.execute_input":"2022-07-24T16:18:06.237521Z","iopub.status.idle":"2022-07-24T16:18:26.443433Z","shell.execute_reply.started":"2022-07-24T16:18:06.237485Z","shell.execute_reply":"2022-07-24T16:18:26.440441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\ntrain_model(model, dataloaders_dict, criterion, optimizer, 1)\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)\ntrain_model(model, dataloaders_dict, criterion, optimizer, 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:18:26.444548Z","iopub.status.idle":"2022-07-24T16:18:26.445518Z","shell.execute_reply.started":"2022-07-24T16:18:26.445211Z","shell.execute_reply":"2022-07-24T16:18:26.445255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"debug = False\ngenerate_new = True\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\ntest_df","metadata":{"papermill":{"duration":0.02504,"end_time":"2022-07-08T14:24:41.073811","exception":false,"start_time":"2022-07-08T14:24:41.048771","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-24T16:18:26.446902Z","iopub.status.idle":"2022-07-24T16:18:26.448016Z","shell.execute_reply.started":"2022-07-24T16:18:26.447707Z","shell.execute_reply":"2022-07-24T16:18:26.447735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_df = pd.DataFrame({\"image_id\" : [\"006388_0\", \"008e5c_0\", \"00c058_0\", \"01adc5_0\", \"01adc5_0\"], \"patient_id\" : [\"006388\", \"008e5c\", \"00c058\", \"01adc5\", \"01adc5\"]})","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:18:26.449396Z","iopub.status.idle":"2022-07-24T16:18:26.450418Z","shell.execute_reply.started":"2022-07-24T16:18:26.450088Z","shell.execute_reply":"2022-07-24T16:18:26.450116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    os.mkdir(\"../test/\")\nexcept:\n    pass\nfor i in tqdm(range(test_df.shape[0])): # tqdm affiche la barre de profression de la boucle for\n    img_id = test_df.iloc[i].image_id\n    try:\n        sz = os.path.getsize(dirs[1] + img_id + \".tif\")\n    except:\n        sz = 1000000000\n    if(sz > 8e8):\n        img = np.zeros((1024,1024,3), np.uint8)\n    else:\n        try:\n            img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (1024, 1024))\n        except:\n            img = np.zeros((1024,1024,3), np.uint8)\n    cv2.imwrite(f\"../test/{img_id}.jpg\", img)\n    del img\n    gc.collect() #optimisation de la mémoire pas trouver l'utiliter","metadata":{"papermill":{"duration":69.477711,"end_time":"2022-07-08T14:25:50.554416","exception":false,"start_time":"2022-07-08T14:24:41.076705","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-24T16:18:26.452076Z","iopub.status.idle":"2022-07-24T16:18:26.452621Z","shell.execute_reply.started":"2022-07-24T16:18:26.452364Z","shell.execute_reply":"2022-07-24T16:18:26.452388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"../test/\", \"../train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        try:\n            image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        except:\n            image = np.zeros((1024,1024,3), np.uint8)\n        label = 0\n        try:\n            if len(image.shape) == 5:\n                image = image.squeeze().transpose(1, 2, 0)\n            image = cv2.resize(image, (1024, 1024)).transpose(2, 0, 1)\n        except:\n            image = np.zeros((3, 1024, 1024))\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        patient_id = self.df.iloc[index].patient_id\n        return image, label, patient_id","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-24T16:18:26.454135Z","iopub.status.idle":"2022-07-24T16:18:26.455099Z","shell.execute_reply.started":"2022-07-24T16:18:26.454834Z","shell.execute_reply":"2022-07-24T16:18:26.454862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, dataloader):\n    model.cuda()\n    model.eval()\n    dataloader = dataloader\n    outputs = []\n    s = nn.Softmax(dim=1)\n    ids = []\n    for item in tqdm(dataloader, leave=False):\n        patient_id = item[2][0]\n        try:\n            images = item[0].cuda().float()\n            ids.append(patient_id)\n            output = model(images)\n            outputs.append(s(output.cpu()[:,:2])[0].detach().numpy())\n        except:\n            ids.append(patient_id)\n            outputs.append(s(torch.tensor([[1, 1]]).float())[0].detach().numpy())\n    return np.array(outputs), ids","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-24T16:18:26.457151Z","iopub.status.idle":"2022-07-24T16:18:26.457786Z","shell.execute_reply.started":"2022-07-24T16:18:26.457539Z","shell.execute_reply":"2022-07-24T16:18:26.457563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_efficientnet_b4', pretrained=True)\nmodel = torch.jit.load('../input/cnn-strip-ai-training/model.pth')\nbatch_size = 1\ntest_loader = DataLoader(\n    ImgDataset(test_df), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-24T16:18:26.46055Z","iopub.status.idle":"2022-07-24T16:18:26.46135Z","shell.execute_reply.started":"2022-07-24T16:18:26.461086Z","shell.execute_reply":"2022-07-24T16:18:26.461109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anss, ids = predict(model, test_loader)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-24T16:18:26.462793Z","iopub.status.idle":"2022-07-24T16:18:26.463956Z","shell.execute_reply.started":"2022-07-24T16:18:26.463684Z","shell.execute_reply":"2022-07-24T16:18:26.46371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prob = pd.DataFrame({\"CE\" : anss[:,0], \"LAA\" : anss[:,1], \"id\" : ids}).groupby(\"id\").mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:18:26.465347Z","iopub.status.idle":"2022-07-24T16:18:26.466567Z","shell.execute_reply.started":"2022-07-24T16:18:26.466228Z","shell.execute_reply":"2022-07-24T16:18:26.466251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:18:26.468032Z","iopub.status.idle":"2022-07-24T16:18:26.469103Z","shell.execute_reply.started":"2022-07-24T16:18:26.468859Z","shell.execute_reply":"2022-07-24T16:18:26.468883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.CE = prob.CE.to_list()\nsubmission.LAA = prob.LAA.to_list()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:18:26.470365Z","iopub.status.idle":"2022-07-24T16:18:26.471429Z","shell.execute_reply.started":"2022-07-24T16:18:26.471164Z","shell.execute_reply":"2022-07-24T16:18:26.471187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:18:26.472619Z","iopub.status.idle":"2022-07-24T16:18:26.473488Z","shell.execute_reply.started":"2022-07-24T16:18:26.47317Z","shell.execute_reply":"2022-07-24T16:18:26.473195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:18:26.475016Z","iopub.status.idle":"2022-07-24T16:18:26.475832Z","shell.execute_reply.started":"2022-07-24T16:18:26.475574Z","shell.execute_reply":"2022-07-24T16:18:26.475599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# création d'un dossier train et test","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train couche externe","metadata":{}},{"cell_type":"code","source":"from random import randrange\nimport numpy as np\nimport pandas as pd \n\nimport os\nfrom os import path\nimport glob\nimport shutil\nimport time\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.image import imread\n\nfrom cv2 import cv2\nfrom cv2 import resize\nimport PIL\nfrom PIL import Image, ImageDraw, ImageOps, ImageFilter\n\nfrom sklearn import cluster\nfrom sklearn import decomposition\nfrom sklearn import metrics\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import learning_curve\nfrom sklearn.utils import shuffle\n\nimport keras\nimport tensorflow as tf\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint","metadata":{"execution":{"iopub.status.busy":"2022-07-27T13:59:35.220538Z","iopub.execute_input":"2022-07-27T13:59:35.221203Z","iopub.status.idle":"2022-07-27T13:59:43.767942Z","shell.execute_reply.started":"2022-07-27T13:59:35.221086Z","shell.execute_reply":"2022-07-27T13:59:43.76666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add our data-augmentation parameters to ImageDataGenerator\n\ntrain_dir = dirs[0]\nvalidation_dir = dirs[1] # test 5 image\n\ntrain_datagen = ImageDataGenerator(rescale=1./1056.\n                                   , rotation_range=20, width_shift_range=0.1,\n                                   height_shift_range=0.1, shear_range=0.1, zoom_range=0.1, horizontal_flip=False)\n\ntest_datagen = ImageDataGenerator(rescale=1.0/1056.)\n\ntrain_generator = train_datagen.flow_from_directory(\n    train_dir, batch_size=20, class_mode='binary', target_size=(1024, 1024))\n\nvalidation_generator = test_datagen.flow_from_directory(\n    validation_dir, batch_size=20, class_mode='binary', target_size=(1024, 1024))\n\nclass_indices=train_generator.class_indices\nclass_indices","metadata":{"execution":{"iopub.status.busy":"2022-07-27T14:10:49.46656Z","iopub.execute_input":"2022-07-27T14:10:49.466972Z","iopub.status.idle":"2022-07-27T14:10:49.786646Z","shell.execute_reply.started":"2022-07-27T14:10:49.466927Z","shell.execute_reply":"2022-07-27T14:10:49.785519Z"},"trusted":true},"execution_count":null,"outputs":[]}]}