{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-31T00:57:30.049402Z","iopub.execute_input":"2022-12-31T00:57:30.04981Z","iopub.status.idle":"2022-12-31T00:57:30.055526Z","shell.execute_reply.started":"2022-12-31T00:57:30.049779Z","shell.execute_reply":"2022-12-31T00:57:30.054441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:57:30.057418Z","iopub.execute_input":"2022-12-31T00:57:30.058012Z","iopub.status.idle":"2022-12-31T00:57:31.106602Z","shell.execute_reply.started":"2022-12-31T00:57:30.057975Z","shell.execute_reply":"2022-12-31T00:57:31.105471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:57:31.10918Z","iopub.execute_input":"2022-12-31T00:57:31.113758Z","iopub.status.idle":"2022-12-31T00:57:40.591569Z","shell.execute_reply.started":"2022-12-31T00:57:31.113688Z","shell.execute_reply":"2022-12-31T00:57:40.590333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import classification_report\nimport timm.optim.optim_factory as optim_factory\nfrom timm.data import create_transform\nfrom torch.utils.data import Dataset\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn import metrics\n%matplotlib inline\nimport torch.nn as nn\nfrom PIL import Image\nimport pandas as pd\nimport numpy as np\nimport torch\nimport timm\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:57:40.594675Z","iopub.execute_input":"2022-12-31T00:57:40.595705Z","iopub.status.idle":"2022-12-31T00:57:40.606649Z","shell.execute_reply.started":"2022-12-31T00:57:40.595656Z","shell.execute_reply":"2022-12-31T00:57:40.605656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pfbeta_torch(preds, labels, beta=1):\n    preds = preds.clip(0, 1)\n\n    y_true_count = labels.sum()\n    ctp = preds[labels == 1].sum()\n    cfp = preds[labels == 0].sum()\n\n    beta_squared = beta * beta\n\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n\n    if c_precision > 0 and c_recall > 0:\n        return ((1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)).item()\n    else:\n        return 0.0\n\ndef pfbeta_thresh(preds, labels):\n    optimized_preds = optimize_preds(preds, labels)\n    return pfbeta_torch(optimized_preds, labels)\n\n\ndef optimize_preds(preds, labels, return_thresh=False, print_results=False):\n    preds = preds.clone()\n\n    without_thresh = pfbeta_torch(preds, labels)\n\n    threshs = np.linspace(0, 1, 101)\n    f1s = [pfbeta_torch((preds > thr).float(), labels) for thr in threshs]\n    idx = np.argmax(f1s)\n    thresh, best_pfbeta = threshs[idx], f1s[idx]\n\n    preds = (preds > thresh).float()\n\n    if print_results:\n        print(f\"without optimization: {without_thresh:.3f}\")\n        pfbeta = pfbeta_torch(preds, labels)\n        print(f\"with optimization: {pfbeta:.3f}\")\n        print(f\"best_thresh: {thresh}\")\n\n    if return_thresh:\n        return thresh\n\n    return preds","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:57:40.61004Z","iopub.execute_input":"2022-12-31T00:57:40.610583Z","iopub.status.idle":"2022-12-31T00:57:40.621206Z","shell.execute_reply.started":"2022-12-31T00:57:40.610545Z","shell.execute_reply":"2022-12-31T00:57:40.620214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_EPOCHS = 4\nNUM_SPLITS = 5\n\nRESIZE_TO = (1024, 512)\n\nbatch_size_train = 8\nbatch_size_test = 2\n# THRESHOLD = 0.7\n\nDATA_PATH = '/kaggle/input/rsna-breast-cancer-detection/'\nTRAIN_IMAGE_DIR = '/kaggle/input/rsna-bcd-roi-1024x-png-dataset/train_images/'\n# TEST_DICOM_DIR = '/kaggle/input/rsna-breast-cancer-detection/test_images/'\n# SAVE_FOLDER = \"/kaggle/tmp/output/\"\n# MODEL_PATH = ''","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:57:40.622844Z","iopub.execute_input":"2022-12-31T00:57:40.623516Z","iopub.status.idle":"2022-12-31T00:57:40.635874Z","shell.execute_reply.started":"2022-12-31T00:57:40.623479Z","shell.execute_reply":"2022-12-31T00:57:40.63477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv(f'{DATA_PATH}/train.csv')\ntrain_csv['path'] = TRAIN_IMAGE_DIR + train_csv[\"patient_id\"].astype(str) + \"/\" + train_csv[\"image_id\"].astype(str) + \".png\"\n\nskf = StratifiedKFold(NUM_SPLITS, shuffle=True, random_state=7)\n\ndata_train=train_csv['path'].values\nlabels_train=train_csv['cancer'].values\n","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:57:40.637698Z","iopub.execute_input":"2022-12-31T00:57:40.638106Z","iopub.status.idle":"2022-12-31T00:57:40.802824Z","shell.execute_reply.started":"2022-12-31T00:57:40.638028Z","shell.execute_reply":"2022-12-31T00:57:40.801771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BalanceSampler(torch.utils.data.sampler.Sampler):\n    def __init__(self, dataset, ratio=3):\n        self.r = ratio-1\n        self.dataset = dataset\n        self.pos_index = np.where(dataset.label>0)[0]\n        self.neg_index = np.where(dataset.label==0)[0]\n\n        self.length = self.r*int(np.floor(len(self.neg_index)/self.r))\n\n    def __iter__(self):\n        pos_index = self.pos_index.copy()\n        neg_index = self.neg_index.copy()\n        np.random.shuffle(pos_index)\n        np.random.shuffle(neg_index)\n\n        neg_index = neg_index[:self.length].reshape(-1,self.r)\n        pos_index = np.random.choice(pos_index, self.length//self.r).reshape(-1,1)\n\n        index = np.concatenate([pos_index,neg_index],-1).reshape(-1)\n        return iter(index)\n\n    def __len__(self):\n        return self.length","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:57:40.804343Z","iopub.execute_input":"2022-12-31T00:57:40.805102Z","iopub.status.idle":"2022-12-31T00:57:40.815971Z","shell.execute_reply.started":"2022-12-31T00:57:40.805046Z","shell.execute_reply":"2022-12-31T00:57:40.814977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GetLoader(Dataset):\n    def __init__(self, transform, data_train=train_csv['path'].values, labels_train=train_csv['cancer'].values, is_train=True):\n        self.data = data_train\n        self.label = labels_train\n        self.trans = transform\n\n    def __getitem__(self, index):\n        data = Image.open(self.data[index]).convert('RGB')\n        data = torch.as_tensor(self.trans(data),dtype=torch.float32).cuda()\n        labels = torch.as_tensor(self.label[index],dtype=torch.float32).cuda()\n        return data, labels\n\n    def __len__(self):\n        return len(self.data)\n\n\ndef build_transform(is_train):\n    if is_train:\n        transform = create_transform(\n            input_size=RESIZE_TO,\n            is_training=True,\n            scale=(0.75, 1.33),\n            ratio=(0.08, 1.0),\n            hflip=0.5,\n            vflip=0.5,\n            color_jitter=0.4,\n            interpolation=\"random\",\n        )\n    else:\n        transform = create_transform(\n            input_size=RESIZE_TO,\n            is_training=False,\n            interpolation=\"bilinear\",\n        )\n    return transform\n\n\ndef build_dataset(data_train=train_csv['path'].values, labels_train=train_csv['cancer'].values, is_train=True):\n    transform = build_transform(is_train)\n    dataset = GetLoader(transform,data_train, labels_train, is_train=True)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:57:40.817625Z","iopub.execute_input":"2022-12-31T00:57:40.81814Z","iopub.status.idle":"2022-12-31T00:57:40.829923Z","shell.execute_reply.started":"2022-12-31T00:57:40.818102Z","shell.execute_reply":"2022-12-31T00:57:40.828966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tf_efficientnetv2_s():\n    model = timm.create_model(\n        'tf_efficientnetv2_s', pretrained=True, in_chans=3, num_classes=1,)\n    return model","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-31T00:57:40.831627Z","iopub.execute_input":"2022-12-31T00:57:40.83202Z","iopub.status.idle":"2022-12-31T00:57:40.843318Z","shell.execute_reply.started":"2022-12-31T00:57:40.831984Z","shell.execute_reply":"2022-12-31T00:57:40.842297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for train_index, test_index in skf.split(data_train,labels_train):\n        X_train, X_test = np.array(data_train)[train_index], np.array(data_train)[test_index]\n        y_train, y_test = np.array(labels_train)[train_index], np.array(labels_train)[test_index]\n        train_dataset = build_dataset(X_train, y_train,is_train=True)\n        train_dataloader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=batch_size_train,\n        sampler=BalanceSampler(train_dataset),\n        drop_last=True,\n    )\n        test_dataset = build_dataset(X_test, y_test,is_train=False)\n        test_dataloader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=batch_size_test,\n        drop_last=False,\n    )\n        model = tf_efficientnetv2_s().cuda()\n        model_without_ddp = model\n        param_groups = optim_factory.param_groups_weight_decay(model_without_ddp,0.05)\n        optimizer = torch.optim.AdamW(param_groups, lr=3e-4, betas=(0.9, 0.999))\n        loss = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([3])).cuda()\n#         loss = nn.BCEWithLogitsLoss().cuda()\n        model.train()\n        for epoch in range(NUM_EPOCHS):\n            model.train()\n            loss_list = []\n            for img, label in tqdm(train_dataloader):\n                label = label.unsqueeze(1)\n                l = loss(model(img),label.float())\n                loss_list.append(l.item())\n                optimizer.zero_grad()\n                l.backward()\n                optimizer.step()\n#                 break\n#             print(\"loss:{}\".format(np.array(loss_list).mean()))","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:57:40.848715Z","iopub.execute_input":"2022-12-31T00:57:40.849645Z","iopub.status.idle":"2022-12-31T00:58:10.010079Z","shell.execute_reply.started":"2022-12-31T00:57:40.849613Z","shell.execute_reply":"2022-12-31T00:58:10.007275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = []\nlabels = []\nmodel.eval()\nfor img, label in tqdm(test_dataloader):\n    output = torch.sigmoid(model(img)).detach().cpu()\n    # output = model(img)\n    prediction.append(output)\n    labels.append(label.cpu()) \n#     break\npreds = torch.cat(prediction)\nlabels = torch.cat(labels)\nthreshold = optimize_preds(preds.float(), labels.float(), return_thresh=True, print_results=True)\nprint(threshold)\ntorch.save(model.state_dict(),'/kaggle/working/Kaggleefficientnet_b4_epoch{}.pth'.format(epoch))","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:11.481558Z","iopub.execute_input":"2022-12-31T00:58:11.481934Z","iopub.status.idle":"2022-12-31T00:58:12.945159Z","shell.execute_reply.started":"2022-12-31T00:58:11.481904Z","shell.execute_reply":"2022-12-31T00:58:12.943028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_dataset = build_dataset(is_train=True)\n# train_dataloader = torch.utils.data.DataLoader(train_dataset,batch_size=1,drop_last=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:10.01815Z","iopub.status.idle":"2022-12-31T00:58:10.021275Z","shell.execute_reply.started":"2022-12-31T00:58:10.02102Z","shell.execute_reply":"2022-12-31T00:58:10.021044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_dataset = build_dataset(is_train=True)\n# train_dataloader = torch.utils.data.DataLoader(train_dataset,batch_size=128,drop_last=True)\n# prediction = []\n# labels = []\n# model.eval()\n# for img, label in tqdm.tqdm(train_dataloader):\n#     output = model(img)\n# #     pred_list = (output.detach().cpu().numpy() > THRESHOLD).astype(int)\n#     pred_list = (output.detach().cpu().numpy())\n#     labe_list = label.cpu().numpy()\n#     prediction = np.append(prediction,pred_list)\n#     labels = np.append(labels,labe_list)\n# print(prediction.shape)\n# print(classification_report(labels,prediction))","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:10.022831Z","iopub.status.idle":"2022-12-31T00:58:10.023631Z","shell.execute_reply.started":"2022-12-31T00:58:10.023384Z","shell.execute_reply":"2022-12-31T00:58:10.023406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pres = [nn.Sigmoid()(torch.tensor(i)).item() for i in prediction]","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:10.025004Z","iopub.status.idle":"2022-12-31T00:58:10.032804Z","shell.execute_reply.started":"2022-12-31T00:58:10.032511Z","shell.execute_reply":"2022-12-31T00:58:10.032534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pres = [nn.Sigmoid()(torch.tensor(i)).item() for i in prediction]\n# THRESHOLD = 0.014\n# pres = [int(i>THRESHOLD) for i in pres]\n# print(classification_report(labels,pres))","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:10.034101Z","iopub.status.idle":"2022-12-31T00:58:10.034815Z","shell.execute_reply.started":"2022-12-31T00:58:10.034572Z","shell.execute_reply":"2022-12-31T00:58:10.034594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pres = [nn.Sigmoid()(torch.tensor(i)).item() for i in prediction]\n# np.absolutearray(pres).max()","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:10.036042Z","iopub.status.idle":"2022-12-31T00:58:10.036784Z","shell.execute_reply.started":"2022-12-31T00:58:10.03653Z","shell.execute_reply":"2022-12-31T00:58:10.036552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train= np.array(data_train)\n# y_train = np.array(labels_train)\n# train_dataset = build_dataset(X_train, y_train,is_train=True)\n# train_dataloader = torch.utils.data.DataLoader(\n# train_dataset,\n# batch_size=batch_size,\n# drop_last=True,\n# )\n# model = efficientnet_b4().cuda()\n# model_without_ddp = model\n# #         print(\"Model = %s\" % str(model_without_ddp))\n# param_groups = optim_factory.param_groups_weight_decay(model_without_ddp,0.05)\n# optimizer = torch.optim.AdamW(param_groups, lr=0.001, betas=(0.9, 0.95))\n# loss = nn.MSELoss()\n# model.train()\n\n# for epoch in range(epochs):\n#     for img, label in tqdm.tqdm(train_dataloader):\n#         label = label.unsqueeze(1)\n#         l = loss(model(img),label.float())\n#         optimizer.zero_grad()\n#         l.backward()\n#         optimizer.step()\n#     print(l)","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:10.038032Z","iopub.status.idle":"2022-12-31T00:58:10.038737Z","shell.execute_reply.started":"2022-12-31T00:58:10.038504Z","shell.execute_reply":"2022-12-31T00:58:10.038526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for train_index, test_index in skf.split(data_train,labels_train):\n#         X_train, X_test = np.array(data_train)[train_index], np.array(data_train)[test_index]\n#         y_train, y_test = np.array(labels_train)[train_index], np.array(labels_train)[test_index]\n#         train_dataset = build_dataset(X_train, y_train,is_train=True)\n#         train_dataloader = torch.utils.data.DataLoader(\n#         train_dataset,\n#         batch_size=batch_size,\n#         drop_last=True,\n#     )\n#         test_dataset = build_dataset(X_test, y_test,is_train=False)\n#         test_dataloader = torch.utils.data.DataLoader(\n#         test_dataset,\n#         batch_size=batch_size,\n#         drop_last=False,\n#     )\n#         model = tf_efficientnetv2_s().cuda()\n#         model_without_ddp = model\n#         param_groups = optim_factory.param_groups_weight_decay(model_without_ddp,0.05)\n#         optimizer = torch.optim.AdamW(param_groups, lr=3e-4, betas=(0.9, 0.999))\n#         loss = nn.BCELoss().cuda()\n#         model.train()\n        \n#         for epoch in range(epochs):\n#             model.train()\n#             for img, label in tqdm.tqdm(train_dataloader):\n#                 label = label.unsqueeze(1)\n#                 l = loss(nn.Sigmoid()(model(img)),label.float())\n#                 optimizer.zero_grad()\n#                 l.backward()\n#                 optimizer.step()\n#             print(\"loss:{}\".format(l))\n#             prediction = []\n#             labels = []\n#             model.eval()\n#             for img, label in test_dataloader:\n#                 output = nn.Sigmoid()(model(img))\n#                 pred_list = (output.detach().cpu().numpy() > THRESHOLD).astype(int)\n#                 labe_list = label.cpu().numpy()\n#                 prediction = np.append(prediction,pred_list)\n#                 labels = np.append(labels,labe_list)\n#             print(prediction.shape)\n#             print(classification_report(labels,prediction))\n#         break","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:10.04004Z","iopub.status.idle":"2022-12-31T00:58:10.040744Z","shell.execute_reply.started":"2022-12-31T00:58:10.04051Z","shell.execute_reply":"2022-12-31T00:58:10.040532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prediction = []\n# labels = []\n# for img, label in test_dataloader:\n#     output = model(img)\n#     pred_list = (output.detach().cpu().numpy() > THRESHOLD).astype(int)\n#     labe_list = label.cpu().numpy()\n#     prediction = np.append(prediction,pred_list)\n#     labels = np.append(labels,labe_list)\n# print(prediction.shape)\n# print(classification_report(labels,prediction))\n# #     break\n# #     output = model(img)\n    \n\n# #     label =label.argmax(dim=1, keepdim=True).flatten()\n# #     pred_list = pred.cpu().numpy()\n# #     labe_list = label.cpu().numpy()\n# #     prediction = np.append(prediction,pred_list)\n# #     labels = np.append(labels,labe_list)\n# # print(prediction.shape)\n# # print(classification_report(labels,prediction))","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:10.042009Z","iopub.status.idle":"2022-12-31T00:58:10.042722Z","shell.execute_reply.started":"2022-12-31T00:58:10.042493Z","shell.execute_reply":"2022-12-31T00:58:10.042515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(),'/kaggle/working/tf_efficientnetv2_s.pth')","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:10.050322Z","iopub.status.idle":"2022-12-31T00:58:10.051085Z","shell.execute_reply.started":"2022-12-31T00:58:10.050849Z","shell.execute_reply":"2022-12-31T00:58:10.050871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:58:10.052314Z","iopub.status.idle":"2022-12-31T00:58:10.053025Z","shell.execute_reply.started":"2022-12-31T00:58:10.052793Z","shell.execute_reply":"2022-12-31T00:58:10.052815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}