{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install pylibjpeg pylibjpeg-libjpeg pydicom --target=/kaggle/working/mysitepackages\n# !pip install gdcm --target=/kaggle/working/mysitepackages","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:03:03.030302Z","iopub.execute_input":"2022-12-30T12:03:03.030668Z","iopub.status.idle":"2022-12-30T12:03:03.046742Z","shell.execute_reply.started":"2022-12-30T12:03:03.030588Z","shell.execute_reply":"2022-12-30T12:03:03.045908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pydicom as dicom\nimport matplotlib.pylab as plt\nimport seaborn as sns\nfrom torch.utils.data.sampler import SubsetRandomSampler,WeightedRandomSampler\nimport sys\nsys.path.append('/kaggle/working/mysitepackages')\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport cv2\nimport os\nimport torch\nfrom torchvision import datasets, transforms,models\nfrom torch.utils.data import Dataset, Subset\nfrom torchvision.utils import make_grid\nfrom torchvision.datasets import ImageFolder\nfrom typing import Tuple,Any\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-30T12:03:03.048452Z","iopub.execute_input":"2022-12-30T12:03:03.049026Z","iopub.status.idle":"2022-12-30T12:03:06.059924Z","shell.execute_reply.started":"2022-12-30T12:03:03.04898Z","shell.execute_reply":"2022-12-30T12:03:06.059041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:03:06.065059Z","iopub.execute_input":"2022-12-30T12:03:06.065911Z","iopub.status.idle":"2022-12-30T12:03:06.194671Z","shell.execute_reply.started":"2022-12-30T12:03:06.065881Z","shell.execute_reply":"2022-12-30T12:03:06.193564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data[train_data['image_id']==1369768762]","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:03:06.197516Z","iopub.execute_input":"2022-12-30T12:03:06.198223Z","iopub.status.idle":"2022-12-30T12:03:06.221797Z","shell.execute_reply.started":"2022-12-30T12:03:06.198182Z","shell.execute_reply":"2022-12-30T12:03:06.218746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.patient_id.nunique(), train_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:03:06.225682Z","iopub.execute_input":"2022-12-30T12:03:06.226108Z","iopub.status.idle":"2022-12-30T12:03:06.243444Z","shell.execute_reply.started":"2022-12-30T12:03:06.226072Z","shell.execute_reply":"2022-12-30T12:03:06.242436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.groupby('patient_id')['image_id'].nunique().sort_values()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:03:06.247375Z","iopub.execute_input":"2022-12-30T12:03:06.247788Z","iopub.status.idle":"2022-12-30T12:03:06.273131Z","shell.execute_reply.started":"2022-12-30T12:03:06.247754Z","shell.execute_reply":"2022-12-30T12:03:06.272133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.groupby(['cancer']).patient_id.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:03:06.277535Z","iopub.execute_input":"2022-12-30T12:03:06.277927Z","iopub.status.idle":"2022-12-30T12:03:06.294268Z","shell.execute_reply.started":"2022-12-30T12:03:06.277887Z","shell.execute_reply":"2022-12-30T12:03:06.293356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def loader(path):\n#     return dicom.read_file(path).pixel_array.astype(np.uint8)\n\n# class Chooser:\n#     def __init__(self):\n#         self._counter = -1\n\n#     def __call__(self, path):\n#         if '.dcm' in path:\n#             return True\n    \nclass CustomImageFolder(ImageFolder):\n    def __getitem__(self, index: int) -> Tuple[Any, Any]:\n        path, target = self.samples[index]\n        #print(path, target,int(os.path.basename(path).split(\"_\")[1].split(\".\")[0]), train_data[train_data['image_id']==int(os.path.basename(path).split(\"_\")[1].split(\".\")[0])])\n        target = train_data[train_data['image_id']==int(os.path.basename(path).split(\"_\")[1].split(\".\")[0])].cancer.values[0]\n        sample = self.loader(path)\n        if self.transform is not None:\n            sample = self.transform(sample)\n        if self.target_transform is not None:\n            target = self.target_transform(target)\n        return sample, target\n            \ntransform = transforms.Compose([#transforms.ToPILImage(),\n                                transforms.Resize(224),\n                                 #transforms.CenterCrop(224),\n                                 transforms.ToTensor()])\n\ndataset = CustomImageFolder('/kaggle/input/rsna-breast-cancer-1024-pngs',transform=transform)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:03:06.298057Z","iopub.execute_input":"2022-12-30T12:03:06.300102Z","iopub.status.idle":"2022-12-30T12:04:04.730669Z","shell.execute_reply.started":"2022-12-30T12:03:06.300069Z","shell.execute_reply":"2022-12-30T12:04:04.729647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def imshow(image, ax=None, title=None, normalize=False):\n    \"\"\"Imshow for Tensor.\"\"\"\n    if ax is None:\n        fig, ax = plt.subplots()\n    image = image.numpy().transpose((1, 2, 0))\n\n    if normalize:\n        mean = np.array([0.485, 0.456, 0.406])\n        std = np.array([0.229, 0.224, 0.225])\n        image = std * image + mean\n        image = np.clip(image, 0, 1)\n\n    ax.imshow(image)\n    ax.spines['top'].set_visible(False)\n    ax.spines['right'].set_visible(False)\n    ax.spines['left'].set_visible(False)\n    ax.spines['bottom'].set_visible(False)\n    ax.tick_params(axis='both', length=0)\n    ax.set_xticklabels('')\n    ax.set_yticklabels('')\n\n    return ax","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:04.734181Z","iopub.execute_input":"2022-12-30T12:04:04.734487Z","iopub.status.idle":"2022-12-30T12:04:04.742321Z","shell.execute_reply.started":"2022-12-30T12:04:04.73446Z","shell.execute_reply":"2022-12-30T12:04:04.741235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imshow(dataset[1][0])\ndataset[1][0].shape,dataset[2][1]","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:04.74693Z","iopub.execute_input":"2022-12-30T12:04:04.747273Z","iopub.status.idle":"2022-12-30T12:04:04.995599Z","shell.execute_reply.started":"2022-12-30T12:04:04.747246Z","shell.execute_reply":"2022-12-30T12:04:04.994761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_sub = train_data.copy()\ntrain_data_sub['image_id_new'] = train_data.apply(lambda row: str(row['patient_id']) + '_' + str(row['image_id']) + '.png',axis=1)\nlist_of_files = sorted(os.listdir('/kaggle/input/rsna-breast-cancer-1024-pngs/output'))\ntrain_data_sub = train_data_sub[train_data_sub['image_id_new'].isin(list_of_files)]\ntrain_data_sub.head()\nprint(train_data_sub.shape, train_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:04.997109Z","iopub.execute_input":"2022-12-30T12:04:04.99746Z","iopub.status.idle":"2022-12-30T12:04:05.77094Z","shell.execute_reply.started":"2022-12-30T12:04:04.997427Z","shell.execute_reply":"2022-12-30T12:04:05.769884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nvalidation_split = .2\nshuffle_dataset = True\nrandom_seed= 42\n\n# Creating data indices for training and validation splits:\ndataset_size = len(dataset)\nindices = list(range(dataset_size))\nsplit = int(np.floor(validation_split * dataset_size))\nif shuffle_dataset :\n    np.random.seed(random_seed)\n    np.random.shuffle(indices)\ntrain_indices, val_indices = indices[split:], indices[:split]\n\nclass_sample_count = torch.tensor(\n    [(torch.Tensor(train_data_sub['cancer'])[train_indices] == t).sum() for t in torch.unique(torch.Tensor(train_data_sub['cancer']), sorted=True)])\n\n\nweight = 1. / class_sample_count.float()\nsamples_weight = torch.tensor([weight[t] for t in train_data_sub['cancer'][train_indices]])\n\n# Creating PT data samplers and loaders:\n\ntrain_dataset = Subset(dataset, train_indices)\nval_dataset = Subset(dataset, val_indices)\n\n#print(len(train_dataset), len(val_dataset))\n#print(train_dataset.indices)\n\ntrain_sampler = WeightedRandomSampler(samples_weight, len(samples_weight))\nvalid_sampler = SubsetRandomSampler(val_indices)\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, num_workers=2,\n                                           sampler=train_sampler)\nvalidation_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,num_workers=2,\n                                                sampler=valid_sampler)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:05.773014Z","iopub.execute_input":"2022-12-30T12:04:05.77397Z","iopub.status.idle":"2022-12-30T12:04:06.031763Z","shell.execute_reply.started":"2022-12-30T12:04:05.773929Z","shell.execute_reply":"2022-12-30T12:04:06.030728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = next(iter(train_loader)) \nprint(\"images-size:\", images.shape)\n\nout = make_grid(images)\nprint(\"out-size:\", out.shape)\n\nimshow(out) #title=[train_dataset.classes[x] for x in labels])","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:06.033088Z","iopub.execute_input":"2022-12-30T12:04:06.033464Z","iopub.status.idle":"2022-12-30T12:04:11.002141Z","shell.execute_reply.started":"2022-12-30T12:04:06.033429Z","shell.execute_reply":"2022-12-30T12:04:10.99948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:11.003886Z","iopub.execute_input":"2022-12-30T12:04:11.004598Z","iopub.status.idle":"2022-12-30T12:04:11.069333Z","shell.execute_reply.started":"2022-12-30T12:04:11.004555Z","shell.execute_reply":"2022-12-30T12:04:11.06823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# net = models.resnet18(pretrained=True)\n\n# for param in net.parameters():\n#     param.requires_grad = False\n\n# criterion = nn.CrossEntropyLoss()\n# optimizer = optim.SGD(net.parameters(), lr=0.0001, momentum=0.9)\n\n# def accuracy(out, labels):\n#     _,pred = torch.max(out, dim=1)\n#     return torch.sum(pred==labels).item()\n\n# num_ftrs = net.fc.in_features\n# net.fc = nn.Linear(num_ftrs, 2)\n\nnet = models.resnet18()\nnum_ftrs = net.fc.in_features\nnet.fc = nn.Linear(num_ftrs, 2)\nnet.load_state_dict(torch.load('/kaggle/input/resnettrained28dec/resnet.pt'))\nnet = net.to(device)\nnet.eval()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:11.071415Z","iopub.execute_input":"2022-12-30T12:04:11.07227Z","iopub.status.idle":"2022-12-30T12:04:15.282441Z","shell.execute_reply.started":"2022-12-30T12:04:11.07223Z","shell.execute_reply":"2022-12-30T12:04:15.281117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_epochs = 0\nprint_every = 10\nvalid_loss_min = np.Inf\nval_loss = []\nval_acc = []\ntrain_loss = []\ntrain_acc = []\ntotal_step = len(train_loader)\nfor epoch in range(1, n_epochs+1):\n    running_loss = 0.0\n    correct = 0\n    total=0\n    print(f'Epoch {epoch}\\n')\n    for batch_idx, (data_, target_) in enumerate(train_loader):\n        rgb_data, target_ = data_.to(device), target_.to(device)\n        optimizer.zero_grad()\n        outputs = net(rgb_data)\n        loss = criterion(outputs, target_)\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item()\n        _,pred = torch.max(outputs, dim=1)\n        correct += torch.sum(pred==target_).item()\n        total += target_.size(0)\n        if (batch_idx) % 20 == 0:\n            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n                   .format(epoch, n_epochs, batch_idx, total_step, loss.item()))\n    train_acc.append(100 * correct / total)\n    train_loss.append(running_loss/total_step)\n    print(f'\\ntrain-loss: {np.mean(train_loss):.4f}, train-acc: {(100 * correct/total):.4f}')\n    batch_loss = 0\n    total_t=0\n    correct_t=0\n    with torch.no_grad():\n        net.eval()\n        for data_t, target_t in (validation_loader):\n            rgb_data, target_t = data_t.to(device), target_t.to(device)\n            outputs_t = net(rgb_data)\n            loss_t = criterion(outputs_t, target_t)\n            batch_loss += loss_t.item()\n            _,pred_t = torch.max(outputs_t, dim=1)\n            correct_t += torch.sum(pred_t==target_t).item()\n            total_t += target_t.size(0)\n        val_acc.append(100 * correct_t/total_t)\n        val_loss.append(batch_loss/len(validation_loader))\n        network_learned = batch_loss < valid_loss_min\n        print(f'validation loss: {np.mean(val_loss):.4f}, validation acc: {(100 * correct_t/total_t):.4f}\\n')\n\n        \n        if network_learned:\n            valid_loss_min = batch_loss\n            torch.save(net.state_dict(), '/kaggle/working/resnet.pt')\n            print('Improvement-Detected, save-model')\n    net.train()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:15.284116Z","iopub.execute_input":"2022-12-30T12:04:15.284563Z","iopub.status.idle":"2022-12-30T12:04:15.301502Z","shell.execute_reply.started":"2022-12-30T12:04:15.284517Z","shell.execute_reply":"2022-12-30T12:04:15.300336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = plt.figure(figsize=(20,10))\n# plt.title(\"Train-Validation Accuracy\")\n# plt.plot(train_acc, label='train')\n# plt.plot(val_acc, label='validation')\n# plt.xlabel('num_epochs', fontsize=12)\n# plt.ylabel('accuracy', fontsize=12)\n# plt.legend(loc='best')","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:15.303332Z","iopub.execute_input":"2022-12-30T12:04:15.303754Z","iopub.status.idle":"2022-12-30T12:04:15.314215Z","shell.execute_reply.started":"2022-12-30T12:04:15.303712Z","shell.execute_reply":"2022-12-30T12:04:15.313208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#read test csv\ntest_data = pd.read_csv(r'/kaggle/input/rsna-breast-cancer-detection/test.csv')\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:15.315932Z","iopub.execute_input":"2022-12-30T12:04:15.316284Z","iopub.status.idle":"2022-12-30T12:04:15.339086Z","shell.execute_reply.started":"2022-12-30T12:04:15.316251Z","shell.execute_reply":"2022-12-30T12:04:15.338174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(f, size=512):\n    patient = f.split('/')[-2]\n    image = f.split('/')[-1][:-4]\n\n    img_read = dicom.dcmread(f)\n    img = img_read.pixel_array\n    img = (img - img.min()) / (img.max() - img.min())\n\n    if img_read.PhotometricInterpretation == \"MONOCHROME1\":\n        img = 1 - img\n\n    img = cv2.resize(img, (size, size))\n    \n    img = (img * 255).astype(np.uint8)\n    \n    img = np.repeat(img[:, :, np.newaxis], 3, axis=2).T\n    \n    return img","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:15.341953Z","iopub.execute_input":"2022-12-30T12:04:15.342251Z","iopub.status.idle":"2022-12-30T12:04:15.349421Z","shell.execute_reply.started":"2022-12-30T12:04:15.342221Z","shell.execute_reply":"2022-12-30T12:04:15.348339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(test_data))","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:15.35086Z","iopub.execute_input":"2022-12-30T12:04:15.351935Z","iopub.status.idle":"2022-12-30T12:04:15.358807Z","shell.execute_reply.started":"2022-12-30T12:04:15.351901Z","shell.execute_reply":"2022-12-30T12:04:15.357703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images = np.empty((len(test_data),3, 224, 224), dtype='float')\nprint(test_images.shape)\nfor i in range(len(test_data)):\n    image_id = test_data.loc[i,'image_id']\n    patient_id = test_data.loc[i,'patient_id']\n    img_path = '/kaggle/input/rsna-breast-cancer-detection/test_images/{}/{}.dcm'.format(patient_id, image_id)\n    \n    image_data = process(img_path, size=224)\n    test_images[i]= image_data\n\nprint(test_images.shape, test_images[0].shape)\ntest = torch.from_numpy(test_images).to(device,dtype=torch.float)\npredictions = net(test).argmax(axis=-1)\npredictions = np.squeeze(predictions).tolist()\npredictions = pd.Series(predictions, name=\"cancer\")\npredictions =  pd.concat([test_data['prediction_id'], predictions], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:06:16.330022Z","iopub.execute_input":"2022-12-30T12:06:16.330586Z","iopub.status.idle":"2022-12-30T12:06:19.022837Z","shell.execute_reply.started":"2022-12-30T12:06:16.330539Z","shell.execute_reply":"2022-12-30T12:06:19.018241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.head()\n#/kaggle/working/resnet.pt","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:06:52.01455Z","iopub.execute_input":"2022-12-30T12:06:52.014912Z","iopub.status.idle":"2022-12-30T12:06:52.024491Z","shell.execute_reply.started":"2022-12-30T12:06:52.01488Z","shell.execute_reply":"2022-12-30T12:06:52.023082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#predictions = predictions.groupby(['prediction_id']).mean().reset_index()\npredictions = predictions.drop_duplicates(subset=\"prediction_id\")\npredictions.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:06:52.025912Z","iopub.execute_input":"2022-12-30T12:06:52.02785Z","iopub.status.idle":"2022-12-30T12:06:52.040426Z","shell.execute_reply.started":"2022-12-30T12:06:52.027815Z","shell.execute_reply":"2022-12-30T12:06:52.039449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:07:32.352818Z","iopub.execute_input":"2022-12-30T12:07:32.353912Z","iopub.status.idle":"2022-12-30T12:07:32.361138Z","shell.execute_reply.started":"2022-12-30T12:07:32.353864Z","shell.execute_reply":"2022-12-30T12:07:32.360116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#predictions.to_csv('test_submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T12:04:23.739582Z","iopub.execute_input":"2022-12-30T12:04:23.740076Z","iopub.status.idle":"2022-12-30T12:04:23.745338Z","shell.execute_reply.started":"2022-12-30T12:04:23.740039Z","shell.execute_reply":"2022-12-30T12:04:23.744409Z"},"trusted":true},"execution_count":null,"outputs":[]}]}