{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Install gdcm and pylibjpeg \n!pip install /kaggle/input/rsna-2022-whl/{pydicom-2.3.0-py3-none-any.whl,pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:41:32.297317Z","iopub.execute_input":"2023-02-28T09:41:32.297999Z","iopub.status.idle":"2023-02-28T09:41:47.670289Z","shell.execute_reply.started":"2023-02-28T09:41:32.297953Z","shell.execute_reply":"2023-02-28T09:41:47.668514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Process training data**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ndf_train = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:41:47.679157Z","iopub.execute_input":"2023-02-28T09:41:47.67973Z","iopub.status.idle":"2023-02-28T09:41:47.861712Z","shell.execute_reply.started":"2023-02-28T09:41:47.679685Z","shell.execute_reply":"2023-02-28T09:41:47.859784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n#trick to avoid notebook out of memory errors in competition submissions\n#courtesy of Mark Wijkhuizen\nhuge_10gb_array = np.arange(int(10 * 2**30 / 8), dtype=np.int64)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:41:47.863517Z","iopub.execute_input":"2023-02-28T09:41:47.863966Z","iopub.status.idle":"2023-02-28T09:41:51.854857Z","shell.execute_reply.started":"2023-02-28T09:41:47.863924Z","shell.execute_reply":"2023-02-28T09:41:51.853359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nimport os\nfrom PIL.Image import fromarray\n\nids = []\ndicom_data = []\ncount = 0\nfor dirName, subdirList, fileList in os.walk(\"/kaggle/input/rsna-breast-cancer-detection/train_images\"):\n    for filename in fileList:\n        if count==300: \n            break\n        if \".dcm\" in filename.lower():\n            dicom_file = pydicom.dcmread(os.path.join(dirName,filename))\n            image = dicom_file.pixel_array.astype(np.float32)\n            image = fromarray(image)\n            image = np.array(image)\n            image = np.resize(image, (300, 300))\n            dicom_data.append(image)\n            ids.append(dirName.split('/')[-1])\n            count += 1        \n        ","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:41:51.85827Z","iopub.execute_input":"2023-02-28T09:41:51.858929Z","iopub.status.idle":"2023-02-28T09:49:28.788951Z","shell.execute_reply.started":"2023-02-28T09:41:51.858886Z","shell.execute_reply":"2023-02-28T09:49:28.786656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ndel huge_10gb_array\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:28.791579Z","iopub.execute_input":"2023-02-28T09:49:28.792048Z","iopub.status.idle":"2023-02-28T09:49:29.001693Z","shell.execute_reply.started":"2023-02-28T09:49:28.792004Z","shell.execute_reply":"2023-02-28T09:49:28.999595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids[0:10]","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:29.003408Z","iopub.execute_input":"2023-02-28T09:49:29.004815Z","iopub.status.idle":"2023-02-28T09:49:29.015372Z","shell.execute_reply.started":"2023-02-28T09:49:29.004762Z","shell.execute_reply":"2023-02-28T09:49:29.013198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\nfor x in ids:\n    labels.append(df_train[df_train.patient_id == int(ids[0])]['cancer'].max())\n    \nlabels[0:10]","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:29.017124Z","iopub.execute_input":"2023-02-28T09:49:29.017777Z","iopub.status.idle":"2023-02-28T09:49:29.261637Z","shell.execute_reply.started":"2023-02-28T09:49:29.017683Z","shell.execute_reply":"2023-02-28T09:49:29.259456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nx = torch.from_numpy(np.array(dicom_data)).reshape(len(ids), -1)\nx.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:29.263867Z","iopub.execute_input":"2023-02-28T09:49:29.26525Z","iopub.status.idle":"2023-02-28T09:49:32.611405Z","shell.execute_reply.started":"2023-02-28T09:49:29.265182Z","shell.execute_reply":"2023-02-28T09:49:32.610296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = torch.from_numpy(np.array(labels)).reshape(len(ids), -1)\ny.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:32.613779Z","iopub.execute_input":"2023-02-28T09:49:32.61453Z","iopub.status.idle":"2023-02-28T09:49:32.625335Z","shell.execute_reply.started":"2023-02-28T09:49:32.614479Z","shell.execute_reply":"2023-02-28T09:49:32.62324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Build model**","metadata":{}},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:32.627181Z","iopub.execute_input":"2023-02-28T09:49:32.627577Z","iopub.status.idle":"2023-02-28T09:49:32.636527Z","shell.execute_reply.started":"2023-02-28T09:49:32.62754Z","shell.execute_reply":"2023-02-28T09:49:32.635093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\n\nmodel = nn.Sequential(\n        nn.Linear(90000, 256),\n        nn.ReLU(inplace=True),\n        nn.Linear(256, 1)\n).to(device)\n\ncriterion = nn.BCELoss()\noptimizer = torch.optim.Adam(model.parameters())\n\nmodel.train()\nfor epoch in range(10):\n    outputs = model(x).sigmoid()\n    loss = criterion(outputs.float(), y.float())\n    print(f\"epoch: {epoch}, loss: {loss}\")\n\n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:32.638096Z","iopub.execute_input":"2023-02-28T09:49:32.638527Z","iopub.status.idle":"2023-02-28T09:49:38.333376Z","shell.execute_reply.started":"2023-02-28T09:49:32.638488Z","shell.execute_reply":"2023-02-28T09:49:38.331711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:38.33604Z","iopub.execute_input":"2023-02-28T09:49:38.336584Z","iopub.status.idle":"2023-02-28T09:49:38.50127Z","shell.execute_reply.started":"2023-02-28T09:49:38.336536Z","shell.execute_reply":"2023-02-28T09:49:38.499834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Predict and submit**","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\")\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:38.504987Z","iopub.execute_input":"2023-02-28T09:49:38.506384Z","iopub.status.idle":"2023-02-28T09:49:38.541214Z","shell.execute_reply.started":"2023-02-28T09:49:38.506331Z","shell.execute_reply":"2023-02-28T09:49:38.539838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"huge_10gb_array = np.arange(int(10 * 2**30 / 8), dtype=np.int64)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:38.542996Z","iopub.execute_input":"2023-02-28T09:49:38.543788Z","iopub.status.idle":"2023-02-28T09:49:42.457861Z","shell.execute_reply.started":"2023-02-28T09:49:38.543744Z","shell.execute_reply":"2023-02-28T09:49:42.456468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = []\ny_preds = []\nfor dirName, subdirList, fileList in os.walk(\"/kaggle/input/rsna-breast-cancer-detection/test_images\"):\n    for filename in fileList:\n        if \".dcm\" in filename.lower():\n            dicom_file = pydicom.dcmread(os.path.join(dirName,filename))\n            image = dicom_file.pixel_array.astype(np.float32)\n            image = fromarray(image)\n            image = np.array(image)\n            image = np.resize(image, (300, 300))\n            x = torch.from_numpy(image).reshape(-1)\n            ids.append(dirName.split('/')[-1])\n            model.eval()\n            with torch.inference_mode(): \n                y_preds.append(model(x).sigmoid())","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:42.459636Z","iopub.execute_input":"2023-02-28T09:49:42.463728Z","iopub.status.idle":"2023-02-28T09:49:44.527436Z","shell.execute_reply.started":"2023-02-28T09:49:42.463625Z","shell.execute_reply":"2023-02-28T09:49:44.525688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del huge_10gb_array\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:49:44.529429Z","iopub.execute_input":"2023-02-28T09:49:44.530183Z","iopub.status.idle":"2023-02-28T09:49:44.744533Z","shell.execute_reply.started":"2023-02-28T09:49:44.530095Z","shell.execute_reply":"2023-02-28T09:49:44.743271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.DataFrame({'prediction_id': df_test.prediction_id, 'cancer': [x.item() for x in y_preds]})\npredictions = df_sub.groupby('prediction_id')['cancer'].max()\nsubmission = pd.DataFrame({'prediction_id': df_test.prediction_id.unique(), 'cancer': predictions.values})\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:50:36.306937Z","iopub.execute_input":"2023-02-28T09:50:36.307433Z","iopub.status.idle":"2023-02-28T09:50:36.339988Z","shell.execute_reply.started":"2023-02-28T09:50:36.307394Z","shell.execute_reply":"2023-02-28T09:50:36.338441Z"},"trusted":true},"execution_count":null,"outputs":[]}]}