{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\niskaggle = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')\n\nif iskaggle:\n    !pip install -Uqq fastai duckduckgo_search","metadata":{"execution":{"iopub.status.busy":"2023-02-21T22:16:40.862879Z","iopub.execute_input":"2023-02-21T22:16:40.863236Z","iopub.status.idle":"2023-02-21T22:16:51.842192Z","shell.execute_reply.started":"2023-02-21T22:16:40.86313Z","shell.execute_reply":"2023-02-21T22:16:51.841294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport torch\nfrom torchvision.io import read_image\nfrom sklearn.model_selection import train_test_split\nimport pydicom\nfrom pydicom.data import get_testdata_file\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport cv2\nimport glob\nfrom tqdm.notebook import tqdm\nfrom joblib import Parallel, delayed\n\nimport math, re, os\nimport numpy as np\nimport tensorflow as tf\n\nfrom fastai.data.all import *\nfrom fastai.vision.all import *\n","metadata":{"execution":{"iopub.status.busy":"2023-02-21T22:17:25.255921Z","iopub.execute_input":"2023-02-21T22:17:25.256225Z","iopub.status.idle":"2023-02-21T22:17:36.525233Z","shell.execute_reply.started":"2023-02-21T22:17:25.256177Z","shell.execute_reply":"2023-02-21T22:17:36.524447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect TPU, return appropriate distribution strategy\n#try:\n#    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n#    print('Running on TPU ', tpu.master())\n#except ValueError:\n#    tpu = None\n\n#if tpu:\n#    tf.config.experimental_connect_to_cluster(tpu)\n#    tf.tpu.experimental.initialize_tpu_system(tpu)\n#    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n#else:\n#    strategy = tf.distribute.get_strategy() \n\n#print(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('rsna-mammography-images-as-pngs')\nprint(GCS_DS_PATH) # what do gcs paths look like?","metadata":{"execution":{"iopub.status.busy":"2023-02-21T22:17:36.528657Z","iopub.execute_input":"2023-02-21T22:17:36.528864Z","iopub.status.idle":"2023-02-21T22:17:36.910537Z","shell.execute_reply.started":"2023-02-21T22:17:36.52884Z","shell.execute_reply":"2023-02-21T22:17:36.909759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv('../input/rsna-breast-cancer-detection/train.csv')\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-21T22:17:36.911596Z","iopub.execute_input":"2023-02-21T22:17:36.911839Z","iopub.status.idle":"2023-02-21T22:17:37.041924Z","shell.execute_reply.started":"2023-02-21T22:17:36.911806Z","shell.execute_reply":"2023-02-21T22:17:37.0411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = pd.read_csv('../input/rsna-breast-cancer-detection/test.csv')\ntest_csv.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-21T22:17:37.043654Z","iopub.execute_input":"2023-02-21T22:17:37.044005Z","iopub.status.idle":"2023-02-21T22:17:37.062781Z","shell.execute_reply.started":"2023-02-21T22:17:37.043968Z","shell.execute_reply":"2023-02-21T22:17:37.06211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\npath = '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_cv2_256/train_images_processed_cv2_256'\nfn2label = {fn: cancer_or_not for fn, cancer_or_not in zip(train_csv['image_id'].astype('str'), train_csv['cancer'])}\n\ndef label_func(path):\n    return fn2label[path.stem]\n\ndblock = DataBlock(\n    blocks = (ImageBlock, CategoryBlock),\n    get_items = get_image_files,\n    get_y = label_func,\n    splitter = RandomSplitter()\n    )\ndsets = dblock.datasets(path)\ndls = dblock.dataloaders(path)","metadata":{"execution":{"iopub.status.busy":"2023-02-21T22:17:40.973024Z","iopub.execute_input":"2023-02-21T22:17:40.973664Z","iopub.status.idle":"2023-02-21T22:20:07.851003Z","shell.execute_reply.started":"2023-02-21T22:17:40.973626Z","shell.execute_reply":"2023-02-21T22:20:07.850213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch()","metadata":{"execution":{"iopub.status.busy":"2023-02-21T22:20:07.852555Z","iopub.execute_input":"2023-02-21T22:20:07.852799Z","iopub.status.idle":"2023-02-21T22:20:09.339192Z","shell.execute_reply.started":"2023-02-21T22:20:07.852768Z","shell.execute_reply":"2023-02-21T22:20:09.338449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = vision_learner(dls, resnet18, metrics=error_rate)\nlearn.fit_one_cycle(1, 1e-2)","metadata":{"execution":{"iopub.status.busy":"2023-02-21T22:20:38.143511Z","iopub.execute_input":"2023-02-21T22:20:38.143798Z","iopub.status.idle":"2023-02-21T22:26:00.315107Z","shell.execute_reply.started":"2023-02-21T22:20:38.143768Z","shell.execute_reply":"2023-02-21T22:26:00.314145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SPLIT = 0\n# MODEL_PATH = '/kaggle/input/rsna-trained-model-weights/tf_effv2_s_208_402/tf_effv2_s_208_402'\n\n# learn.save(f'{MODEL_PATH}/{SPLIT}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.show_results()","metadata":{"execution":{"iopub.status.busy":"2023-02-21T22:26:12.41628Z","iopub.execute_input":"2023-02-21T22:26:12.416561Z","iopub.status.idle":"2023-02-21T22:26:13.883807Z","shell.execute_reply.started":"2023-02-21T22:26:12.416531Z","shell.execute_reply":"2023-02-21T22:26:13.883184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Trying to make submission file","metadata":{}},{"cell_type":"code","source":"RESIZE_TO = (1024, 1024)\nNUM_SPLITS = 4","metadata":{"execution":{"iopub.status.busy":"2023-02-21T22:58:29.256055Z","iopub.execute_input":"2023-02-21T22:58:29.256353Z","iopub.status.idle":"2023-02-21T22:58:29.260496Z","shell.execute_reply.started":"2023-02-21T22:58:29.256323Z","shell.execute_reply":"2023-02-21T22:58:29.259803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npreds_all = []\n\ntest_dl = learn.dls.test_dl(get_image_files(f'test_resized_{RESIZE_TO[0]}'))\n# for SPLIT in range(NUM_SPLITS):\n    # learn.load(f'{MODEL_PATH}/{SPLIT}')\npreds, _ = learn.get_preds(dl=test_dl)\npreds_all.append(preds)","metadata":{"execution":{"iopub.status.busy":"2023-02-21T23:00:12.836078Z","iopub.execute_input":"2023-02-21T23:00:12.836843Z","iopub.status.idle":"2023-02-21T23:00:12.87713Z","shell.execute_reply.started":"2023-02-21T23:00:12.836807Z","shell.execute_reply":"2023-02-21T23:00:12.876201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = torch.zeros_like(preds_all[0])\nfor pred in preds_all:\n    preds += pred\n\npreds /= NUM_SPLITS\n\n\n# preds = optimize_preds(preds, thresh=threshold)\nimage_ids = [path.stem for path in test_dl.items]\n\nimage_id2pred = defaultdict(lambda: 0)\nfor image_id, pred in zip(image_ids, preds[:, 1]):\n    image_id2pred[int(image_id)] = pred.item()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\n\nprediction_ids = []\npreds = []\n\nfor _, row in test_csv.iterrows():\n    prediction_ids.append(row.prediction_id)\n    preds.append(image_id2pred[row.image_id])\n\nsubmission = pd.DataFrame(data={'prediction_id': prediction_ids,'cancer': preds}).groupby('prediction_id').max().reset_index()\nsubmission.head()","metadata":{},"execution_count":null,"outputs":[]}]}