{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from scipy.io import loadmat\nimport numpy as np\nfrom PIL import Image\nimport cv2\nfrom matplotlib.pyplot import imshow\nimport matplotlib.image as mpimg\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, datasets, models\nimport os\nimport glob\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom collections import defaultdict\nfrom datetime import datetime\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport pandas as pd\nfrom PIL import ImageFile\nImageFile.LOAD_TRUNCATED_IMAGES=True\nImage.MAX_IMAGE_PIXELS = None\nImage_path='../input/backgroundremovedversion/*'\nlabel_path='../input/mayo-clinic-strip-ai/train.csv'\ntrans_preprocess = transforms.Compose([\n    transforms.Resize([512,512]),\n])\nfor i in glob.glob('../input/backgroundremovedversion/*'):\n    \n    for j in glob.glob(f'{i}/*'):\n        img=Image.open(j)\n        img.load()\n        img=trans_preprocess(img)\n        img_data=img.getdata()\n        img=np.array(img_data)\n        split=j.split('/')[-1]\n        p=f'./{split}'\n        cv2.imwrite(p,img)\n             \n  ","metadata":{"execution":{"iopub.status.busy":"2022-10-02T09:57:16.604858Z","iopub.execute_input":"2022-10-02T09:57:16.605219Z","iopub.status.idle":"2022-10-02T10:39:03.013203Z","shell.execute_reply.started":"2022-10-02T09:57:16.60519Z","shell.execute_reply":"2022-10-02T10:39:03.011982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df=pd.read_csv(label_path)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:30:39.642841Z","iopub.execute_input":"2022-10-01T07:30:39.643405Z","iopub.status.idle":"2022-10-01T07:30:39.658403Z","shell.execute_reply.started":"2022-10-01T07:30:39.643367Z","shell.execute_reply":"2022-10-01T07:30:39.657587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trans = transforms.Compose([\n#     transforms.Resize([512,512]),\n#     transforms.RandomHorizontalFlip(),\n#     transforms.ToTensor(),\n#     transforms.Normalize([0.790595  , 0.67119867, 0.8091853 ], [0.05493367, 0.05985045, 0.04747279]) \n# ])\n\n# class Dataset(Dataset):\n#         def __init__(self, Image_path, label_path, transform):\n#             df=pd.read_csv(label_path)\n#             a=df['image_id'].tolist()\n#             image_ls=[]\n#             isin_list=[]\n#             for i in glob.glob(Image_path):\n#                 for j in glob.glob(f'{i}/*'):\n#                     l=[]\n#                     l.append(j)\n#                     split=j.split('/')[-1]\n#                     output=split.split('-')[0]\n#                     l.append(output)\n#                     image_ls.append(l)\n#                     isin_list.append(output)\n#             sorted(image_ls,key=lambda s: s[1])     \n#             sorted(isin_list)\n#             label_ls=[]\n#             for i in isin_list:\n#                 if i in a:\n#                     label_ls.append(df[df['image_id']==i]['label'])\n#             self.img_ls=image_ls\n#             self.label_ls=label_ls\n        \n#         def __len__(self):\n#             return len(self.img_ls)\n#         def __getitem__(self, idx):\n#             img_name = self.img_ls[idx]\n#             img=Image.open(img_name[0])\n#             img.load()\n#             img = self.transform(img)\n#             label = self.label_ls[idx]\n#             return img,label","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:30:39.659896Z","iopub.execute_input":"2022-10-01T07:30:39.660315Z","iopub.status.idle":"2022-10-01T07:30:39.666972Z","shell.execute_reply.started":"2022-10-01T07:30:39.66028Z","shell.execute_reply":"2022-10-01T07:30:39.665871Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image_ls=[]\n# isin_list=[]\n# label_ls=[]\n# label_list=[]\n# to_check=df['image_id'].tolist()\n# for i in glob.glob('../input/backgroundremovedversion/*'):\n    \n#     for j in glob.glob(f'{i}/*'):\n#         l=[]\n#         l.append(j)\n#         split=j.split('/')[-1]\n#         output=split.split('-')[0]\n#         l.append(output)\n#         image_ls.append(l)\n#         isin_list.append(output)\n# image_ls=sorted(image_ls,key=lambda s: s[1])     \n# isin_list=sorted(isin_list) \n# for a in isin_list:\n#     label_ls.extend(df[df['image_id']==a]['label'].tolist())\n# for i in label_ls:\n#     if i ==\"CE\":\n#         label_list.append(0)\n#     else:\n#         label_list.append(1) \n# label_list=torch.Tensor(label_list)\n# image_list=[]\n# for i in image_ls:\n#     image_list.append(i[0])\n# trans_train = transforms.Compose([\n#     transforms.Resize([512,512]),\n#     transforms.RandomHorizontalFlip(),\n#     transforms.ToTensor(),\n#     transforms.Normalize([0.485,0.456,0.406 ], [0.229, 0.224, 0.225]) \n# ])\n# trans_test = transforms.Compose([\n#     transforms.Resize([512,512]),\n#     transforms.ToTensor(),\n#     transforms.Normalize([0.485,0.456,0.406 ], [0.229, 0.224, 0.225]) \n# ])\n# class Dataset(Dataset):\n#         def __init__(self, image_ls, label_ls,transform):\n#             self.img_ls=image_ls\n#             self.label_ls=label_ls\n#             self.transform=transform\n#         def __len__(self):\n#             return len(self.img_ls)\n#         def __getitem__(self, idx):\n#             img_name = self.img_ls[idx]\n#             img=Image.open(img_name) # from disk to cpu ram\n#             img.load()\n#             label = self.label_ls[idx]\n#             img = self.transform(img)\n#             return img,label\n        \n        \n# class D:\n#     def __init__(self, image_ls, labels_ls):\n#         self.images = [load(n) for n in image_ls] # from disk to CPU ram\n#         self.labels = labels_ls\n        \n#     def __get_items(self, idx):\n#         return self.images[idx], self.labels[idx]","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:30:39.670238Z","iopub.execute_input":"2022-10-01T07:30:39.670656Z","iopub.status.idle":"2022-10-01T07:30:40.147815Z","shell.execute_reply.started":"2022-10-01T07:30:39.670619Z","shell.execute_reply":"2022-10-01T07:30:40.146842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# base_model = models.resnet18(pretrained=True)\n# num_features=base_model.fc.in_features\n# base_model.fc=nn.Linear(num_features,2)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:30:40.149266Z","iopub.execute_input":"2022-10-01T07:30:40.14968Z","iopub.status.idle":"2022-10-01T07:30:45.105766Z","shell.execute_reply.started":"2022-10-01T07:30:40.149645Z","shell.execute_reply":"2022-10-01T07:30:45.104648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# lr = 0.001  \n# batch_size = 8 \n# test_image_ls=image_list[615:715]\n# test_label_ls=label_list[615:715]\n# train_image_ls=image_list[0:614]\n# train_label_ls=label_list[0:614]\n# train_set = Dataset(train_image_ls,train_label_ls,transform=trans_train)\n# train_loader = DataLoader(train_set, batch_size=batch_size,shuffle=True)\n# test_set = Dataset(test_image_ls,test_label_ls,transform=trans_test)\n# test_loader = DataLoader(test_set, batch_size=1,shuffle=False)\n# print(device)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:30:45.110896Z","iopub.execute_input":"2022-10-01T07:30:45.11342Z","iopub.status.idle":"2022-10-01T07:30:45.18523Z","shell.execute_reply.started":"2022-10-01T07:30:45.11338Z","shell.execute_reply":"2022-10-01T07:30:45.18411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from torch.utils.tensorboard import SummaryWriter\n# model = base_model\n# model=model.to(device)\n\n# optimizer = torch.optim.Adam(model.parameters(), lr=5e-5)\n\n# loss_fn = F.cross_entropy\n# def train_one_epoch(epoch_index, tb_writer):\n#     running_loss = 0.\n#     last_loss = 0.\n\n#     for i, data in enumerate(train_loader):\n#         # Every data instance is an input + label pair\n#         inputs, labels = data\n#         inputs=inputs.to(device)\n#         labels=labels.to(device)\n#         import pdb; pdb.set_trace()\n#         # Zero your gradients for every batch!\n#         optimizer.zero_grad()\n\n#         # Make predictions for this batch\n#         outputs = model(inputs)\n\n#         # Compute the loss and its gradients\n#         loss = loss_fn(outputs, labels.long())\n#         loss.backward()\n\n#         # Adjust learning weights\n#         optimizer.step()\n\n#         # Gather data and report\n#         running_loss += loss.item()\n#         if i % 1000 == 999:\n#             last_loss = running_loss / 1000 # loss per batch\n#             print('  batch {} loss: {}'.format(i + 1, last_loss))\n#             tb_x = epoch_index * len(training_loader) + i + 1\n#             tb_writer.add_scalar('Loss/train', last_loss, tb_x)\n#             running_loss = 0.\n\n#     return last_loss\n# timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')\n# writer = SummaryWriter('runs/fashion_trainer_{}'.format(timestamp))\n# epoch_number = 0\n\n# EPOCHS = 5\n\n# best_vloss = 1_000_000.\n\n# for epoch in range(EPOCHS):\n#     print('EPOCH {}:'.format(epoch_number + 1))\n#     model.train(True)\n#     avg_loss = train_one_epoch(epoch_number, writer)\n\n#     # We don't need gradients on to do reporting\n#     model.train(False)\n\n#     running_vloss = 0.0\n#     for i, vdata in enumerate(test_loader):\n#         vinputs, vlabels = vdata\n#         vinputs=vinputs.to(device)\n#         vlabels=vlabels.to(device)\n#         voutputs = model(vinputs)\n#         vloss = loss_fn(voutputs, vlabels.long())\n#         running_vloss += vloss\n\n#     avg_vloss = running_vloss / (i + 1)\n#     print('LOSS train {} valid {}'.format(avg_loss, avg_vloss))\n\n#     # Log the running loss averaged per batch\n#     # for both training and validation\n#     writer.add_scalars('Training vs. Validation Loss',\n#                     { 'Training' : avg_loss, 'Validation' : avg_vloss },\n#                     epoch_number + 1)\n#     writer.flush()\n\n#     # Track best performance, and save the model's state\n#     if avg_vloss < best_vloss:\n#         best_vloss = avg_vloss\n#         model_path = 'model_{}_{}'.format(timestamp, epoch_number)\n#         torch.save(model.state_dict(), model_path)\n\n#     epoch_number += 1","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:35:56.770183Z","iopub.execute_input":"2022-10-01T07:35:56.771345Z","iopub.status.idle":"2022-10-01T07:45:37.398312Z","shell.execute_reply.started":"2022-10-01T07:35:56.771294Z","shell.execute_reply":"2022-10-01T07:45:37.396282Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]}]}