{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-16T14:48:15.798689Z","iopub.execute_input":"2022-12-16T14:48:15.79916Z","iopub.status.idle":"2022-12-16T14:48:15.828159Z","shell.execute_reply.started":"2022-12-16T14:48:15.799063Z","shell.execute_reply":"2022-12-16T14:48:15.826745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read in the csv data","metadata":{}},{"cell_type":"code","source":"path='/kaggle/input/rsna-breast-cancer-detection/'\ntrain_data=pd.read_csv(path+'train.csv')\ntest_data=pd.read_csv(path+'test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:49:23.270501Z","iopub.execute_input":"2022-12-16T14:49:23.272245Z","iopub.status.idle":"2022-12-16T14:49:23.449741Z","shell.execute_reply.started":"2022-12-16T14:49:23.272181Z","shell.execute_reply":"2022-12-16T14:49:23.44867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"See what the data looks like","metadata":{}},{"cell_type":"code","source":"print('Train csv')\nprint(train_data.head())\nprint('Test csv')\nprint(test_data)\nprint(train_data.shape)\nprint(test_data.shape)\nprint(test_data.columns)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:49:26.079448Z","iopub.execute_input":"2022-12-16T14:49:26.080026Z","iopub.status.idle":"2022-12-16T14:49:26.11859Z","shell.execute_reply.started":"2022-12-16T14:49:26.079976Z","shell.execute_reply":"2022-12-16T14:49:26.115798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look at overall statistics of data","metadata":{}},{"cell_type":"code","source":"train_data.describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:49:37.061705Z","iopub.execute_input":"2022-12-16T14:49:37.062182Z","iopub.status.idle":"2022-12-16T14:49:37.151681Z","shell.execute_reply.started":"2022-12-16T14:49:37.062144Z","shell.execute_reply":"2022-12-16T14:49:37.149511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:49:43.414156Z","iopub.execute_input":"2022-12-16T14:49:43.414623Z","iopub.status.idle":"2022-12-16T14:49:43.422529Z","shell.execute_reply.started":"2022-12-16T14:49:43.414575Z","shell.execute_reply":"2022-12-16T14:49:43.420506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look at distribution of target variable","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.hist(train_data['cancer'])","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:49:48.192734Z","iopub.execute_input":"2022-12-16T14:49:48.193224Z","iopub.status.idle":"2022-12-16T14:49:48.483814Z","shell.execute_reply.started":"2022-12-16T14:49:48.193187Z","shell.execute_reply":"2022-12-16T14:49:48.482562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Very large number of negative cases compared to positive cases. Therefore, will need to augment the cancerous cases later prior to training an ML model","metadata":{}},{"cell_type":"markdown","source":"Look at pairwise relationships between variables","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nplt.subplot(331)\nplt.title('Cancer vs Age')\nplt.scatter(train_data['age'],train_data['cancer'])\nplt.ylabel('Cancer')\nplt.xlabel('Age')\nplt.subplot(332)\nplt.title('Cancer vs Biopsy')\nplt.scatter(train_data['biopsy'],train_data['cancer'])\nplt.ylabel('Cancer')\nplt.xlabel('Biopsy')\nplt.subplot(333)\nplt.title('Cancer vs Invasive')\nplt.scatter(train_data['invasive'],train_data['cancer'])\nplt.ylabel('Cancer')\nplt.xlabel('Invasive')\nplt.subplot(334)\nplt.title('Cancer vs BIRADS')\nplt.scatter(train_data['BIRADS'],train_data['cancer'])\nplt.ylabel('Cancer')\nplt.xlabel('BIRADS')\nplt.subplot(335)\nplt.title('Cancer vs implant')\nplt.scatter(train_data['implant'],train_data['cancer'])\nplt.ylabel('Cancer')\nplt.xlabel('implant')\nplt.subplot(336)\nplt.title('Cancer vs machine_id')\nplt.scatter(train_data['machine_id'],train_data['cancer'])\nplt.ylabel('Cancer')\nplt.xlabel('machine_id')\nplt.subplot(337)\nplt.title('Cancer vs patient_id')\nplt.scatter(train_data['patient_id'],train_data['cancer'])\nplt.ylabel('Cancer')\nplt.xlabel('patient_id')\nplt.subplot(338)\nplt.title('Cancer vs image_id')\nplt.scatter(train_data['image_id'],train_data['cancer'])\nplt.ylabel('Cancer')\nplt.xlabel('image_id')\nplt.subplot(339)\nplt.title('Cancer vs site_id')\nplt.scatter(train_data['site_id'],train_data['cancer'])\nplt.ylabel('Cancer')\nplt.xlabel('site_id')\n","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:49:53.410935Z","iopub.execute_input":"2022-12-16T14:49:53.41142Z","iopub.status.idle":"2022-12-16T14:49:56.921432Z","shell.execute_reply.started":"2022-12-16T14:49:53.411381Z","shell.execute_reply":"2022-12-16T14:49:56.919767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No apparent correlation between cancer and any of the other input variables in the training dataset. Next, should explore the images","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport cv2\ntrain_images = sorted(glob.glob(\"../input/rsna-mammography-images-as-pngs/images_as_pngs_1024/train_images_processed_1024/*/*\"))","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:50:06.613255Z","iopub.execute_input":"2022-12-16T14:50:06.613743Z","iopub.status.idle":"2022-12-16T14:51:40.168095Z","shell.execute_reply.started":"2022-12-16T14:50:06.6137Z","shell.execute_reply":"2022-12-16T14:51:40.166822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images[0:10]\n#print(len(train_images))","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:52:45.2559Z","iopub.execute_input":"2022-12-16T14:52:45.256416Z","iopub.status.idle":"2022-12-16T14:52:45.266164Z","shell.execute_reply.started":"2022-12-16T14:52:45.256379Z","shell.execute_reply":"2022-12-16T14:52:45.264296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['path']=train_images","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:52:48.800958Z","iopub.execute_input":"2022-12-16T14:52:48.80142Z","iopub.status.idle":"2022-12-16T14:52:48.819824Z","shell.execute_reply.started":"2022-12-16T14:52:48.801386Z","shell.execute_reply":"2022-12-16T14:52:48.818398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.path[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:52:51.569915Z","iopub.execute_input":"2022-12-16T14:52:51.570354Z","iopub.status.idle":"2022-12-16T14:52:51.580321Z","shell.execute_reply.started":"2022-12-16T14:52:51.570317Z","shell.execute_reply":"2022-12-16T14:52:51.578679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.patient_id.unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T11:39:13.620857Z","iopub.execute_input":"2022-12-14T11:39:13.621301Z","iopub.status.idle":"2022-12-14T11:39:13.63213Z","shell.execute_reply.started":"2022-12-14T11:39:13.621266Z","shell.execute_reply":"2022-12-14T11:39:13.631321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read in one image","metadata":{}},{"cell_type":"code","source":"img = cv2.imread(train_images[0], cv2.IMREAD_GRAYSCALE)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:53:00.312799Z","iopub.execute_input":"2022-12-16T14:53:00.314358Z","iopub.status.idle":"2022-12-16T14:53:00.355724Z","shell.execute_reply.started":"2022-12-16T14:53:00.314289Z","shell.execute_reply":"2022-12-16T14:53:00.354208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look at max and min value of image","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimg.shape\nprint(np.min(img))\nprint(np.max(img))","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:53:03.620258Z","iopub.execute_input":"2022-12-16T14:53:03.620793Z","iopub.status.idle":"2022-12-16T14:53:03.631627Z","shell.execute_reply.started":"2022-12-16T14:53:03.620749Z","shell.execute_reply":"2022-12-16T14:53:03.629885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Need to normalise values to be between 0 and 1","metadata":{}},{"cell_type":"markdown","source":"Show image","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nprint(img.shape)\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:53:06.923361Z","iopub.execute_input":"2022-12-16T14:53:06.923804Z","iopub.status.idle":"2022-12-16T14:53:07.311513Z","shell.execute_reply.started":"2022-12-16T14:53:06.923769Z","shell.execute_reply":"2022-12-16T14:53:07.31045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_cancer_1=train_data[train_data['cancer']==1]\ntrain_data_cancer_0=train_data[train_data['cancer']==0]\nprint(train_data_cancer_1.shape)\nprint(train_data_cancer_0.shape)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:53:13.648123Z","iopub.execute_input":"2022-12-16T14:53:13.648562Z","iopub.status.idle":"2022-12-16T14:53:13.678673Z","shell.execute_reply.started":"2022-12-16T14:53:13.648527Z","shell.execute_reply":"2022-12-16T14:53:13.677226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cancer_1_idx=train_data_cancer_1.index\ncancer_0_idx=train_data_cancer_0.index","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:53:16.990041Z","iopub.execute_input":"2022-12-16T14:53:16.99078Z","iopub.status.idle":"2022-12-16T14:53:16.998271Z","shell.execute_reply.started":"2022-12-16T14:53:16.990719Z","shell.execute_reply":"2022-12-16T14:53:16.996745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(cancer_1_idx))\nprint(len(cancer_0_idx))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T13:28:53.208377Z","iopub.execute_input":"2022-12-15T13:28:53.208728Z","iopub.status.idle":"2022-12-15T13:28:53.214409Z","shell.execute_reply.started":"2022-12-15T13:28:53.208701Z","shell.execute_reply":"2022-12-15T13:28:53.213031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_cancer_1.path","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:53:33.816137Z","iopub.execute_input":"2022-12-16T14:53:33.816596Z","iopub.status.idle":"2022-12-16T14:53:33.827447Z","shell.execute_reply.started":"2022-12-16T14:53:33.816558Z","shell.execute_reply":"2022-12-16T14:53:33.825799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define an image generator to read in images","metadata":{}},{"cell_type":"code","source":"def image_gen(img_paths, img_size=(1024, 1024)):\n    # Iterate over all the image paths\n    for path in img_paths:\n        \n        # Load the image and mask, and normalize it to 0-1 range\n        #img = imread(img_path) / 255.\n        img=cv2.imread(path)/255\n        \n        yield img","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:53:41.789096Z","iopub.execute_input":"2022-12-16T14:53:41.790264Z","iopub.status.idle":"2022-12-16T14:53:41.797728Z","shell.execute_reply.started":"2022-12-16T14:53:41.790211Z","shell.execute_reply":"2022-12-16T14:53:41.796279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test if the image generator works","metadata":{}},{"cell_type":"code","source":"test=image_gen(train_data_cancer_0.path)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:53:46.39612Z","iopub.execute_input":"2022-12-16T14:53:46.396569Z","iopub.status.idle":"2022-12-16T14:53:46.404073Z","shell.execute_reply.started":"2022-12-16T14:53:46.396532Z","shell.execute_reply":"2022-12-16T14:53:46.402567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"first_batch = next(test)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:53:48.729269Z","iopub.execute_input":"2022-12-16T14:53:48.729686Z","iopub.status.idle":"2022-12-16T14:53:48.790448Z","shell.execute_reply.started":"2022-12-16T14:53:48.729655Z","shell.execute_reply":"2022-12-16T14:53:48.789232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"first_batch.shape\nplt.imshow(first_batch)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:53:52.833973Z","iopub.execute_input":"2022-12-16T14:53:52.834395Z","iopub.status.idle":"2022-12-16T14:53:53.53051Z","shell.execute_reply.started":"2022-12-16T14:53:52.834363Z","shell.execute_reply":"2022-12-16T14:53:53.529277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Write a generator to get a batch of images at a time","metadata":{}},{"cell_type":"code","source":"def image_batch_generator(img_paths, batchsize=32):\n    while True:\n        ig = image_gen(img_paths)\n        batch_img = []\n        \n        for img in ig:\n            # Add the image and mask to the batch\n            batch_img.append(img)\n            # If we've reached our batchsize, yield the batch and reset\n            if len(batch_img) == batchsize:\n                yield np.stack(batch_img, axis=0)\n                batch_img = []\n        \n        # If we have an nonempty batch left, yield it out and reset\n        if len(batch_img) != 0:\n            yield np.stack(batch_img, axis=0)\n            batch_img = []","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:54:01.574349Z","iopub.execute_input":"2022-12-16T14:54:01.57485Z","iopub.status.idle":"2022-12-16T14:54:01.584333Z","shell.execute_reply.started":"2022-12-16T14:54:01.574812Z","shell.execute_reply":"2022-12-16T14:54:01.582571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batchsize=128\nmy_gen_cancer_0=image_batch_generator(train_data_cancer_0.path,batchsize=batchsize)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:13:23.77356Z","iopub.execute_input":"2022-12-14T13:13:23.774003Z","iopub.status.idle":"2022-12-14T13:13:23.779666Z","shell.execute_reply.started":"2022-12-14T13:13:23.773965Z","shell.execute_reply":"2022-12-14T13:13:23.778254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_count=int(len(train_data_cancer_0.path)/batchsize)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:14:15.383834Z","iopub.execute_input":"2022-12-14T13:14:15.384379Z","iopub.status.idle":"2022-12-14T13:14:15.388604Z","shell.execute_reply.started":"2022-12-14T13:14:15.384349Z","shell.execute_reply":"2022-12-14T13:14:15.387933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(batch_count)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:14:22.125666Z","iopub.execute_input":"2022-12-14T13:14:22.126692Z","iopub.status.idle":"2022-12-14T13:14:22.132018Z","shell.execute_reply.started":"2022-12-14T13:14:22.126651Z","shell.execute_reply":"2022-12-14T13:14:22.130639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read in saved lists","metadata":{}},{"cell_type":"markdown","source":"Code below will read in images from scratch","metadata":{}},{"cell_type":"code","source":"min_list=[]\nmax_list=[]\nmean_cancer_0=[]\nmedian_cancer_0=[]\npercentile_25_cancer_0=[]\npercentile_75_cancer_0=[]\n\n# int(len(train_data_cancer_0.path)/32)\n\nfor i in range(0,batch_count):\n    print('calculating statistics for batch '+str(i))\n    batch=next(my_gen_cancer_0)\n    for j in range(0,batchsize):\n        min_list.append(np.min(batch[j]))\n        max_list.append(np.max(batch[j]))\n        mean_cancer_0.append(np.mean(batch[j]))\n        median_cancer_0.append(np.median(batch[j]))\n        percentile_25_cancer_0.append(np.percentile(batch[j],25))\n        percentile_75_cancer_0.append(np.percentile(batch[j],75))","metadata":{"execution":{"iopub.status.busy":"2022-12-14T13:17:59.411384Z","iopub.execute_input":"2022-12-14T13:17:59.412644Z","iopub.status.idle":"2022-12-14T14:19:49.279578Z","shell.execute_reply.started":"2022-12-14T13:17:59.412591Z","shell.execute_reply":"2022-12-14T14:19:49.277497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(min_list))\nprint(batchsize*413)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T14:20:33.818389Z","iopub.execute_input":"2022-12-14T14:20:33.818875Z","iopub.status.idle":"2022-12-14T14:20:33.824457Z","shell.execute_reply.started":"2022-12-14T14:20:33.818838Z","shell.execute_reply":"2022-12-14T14:20:33.823497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nwith open(\"min_cancer_0\", \"wb\") as fp:   #Pickling\n    pickle.dump(min_list, fp)\nwith open(\"max_cancer_0\", \"wb\") as fp:   #Pickling\n    pickle.dump(max_list, fp)\nwith open(\"mean_cancer_0\", \"wb\") as fp:   #Pickling\n    pickle.dump(mean_cancer_0, fp)\nwith open(\"median_cancer_0\", \"wb\") as fp:   #Pickling\n    pickle.dump(median_cancer_0, fp)\nwith open(\"percentile_25_cancer_0\", \"wb\") as fp:   #Pickling\n    pickle.dump(percentile_25_cancer_0, fp)\nwith open(\"percentile_75_cancer_0\", \"wb\") as fp:   #Pickling\n    pickle.dump(percentile_75_cancer_0, fp)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:54:19.63411Z","iopub.execute_input":"2022-12-16T14:54:19.63458Z","iopub.status.idle":"2022-12-16T14:54:19.715599Z","shell.execute_reply.started":"2022-12-16T14:54:19.634542Z","shell.execute_reply":"2022-12-16T14:54:19.713758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\npath_cancer_0=\"/kaggle/input/image-stats-cancer-0/\"","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:54:30.203743Z","iopub.execute_input":"2022-12-16T14:54:30.204259Z","iopub.status.idle":"2022-12-16T14:54:30.211333Z","shell.execute_reply.started":"2022-12-16T14:54:30.204219Z","shell.execute_reply":"2022-12-16T14:54:30.209594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(path_cancer_0+\"min_cancer_0\", \"rb\") as fp:   \n    min_cancer_0 = pickle.load(fp)\nwith open(path_cancer_0+\"max_cancer_0\", \"rb\") as fp:   \n    max_cancer_0 = pickle.load(fp)\nwith open(path_cancer_0+\"mean_cancer_0\", \"rb\") as fp:   \n    mean_cancer_0 = pickle.load(fp)\nwith open(path_cancer_0+\"median_cancer_0\", \"rb\") as fp:   \n    median_cancer_0 = pickle.load(fp)\nwith open(path_cancer_0+\"percentile_25_cancer_0\", \"rb\") as fp:   \n    percentile_25_cancer_0 = pickle.load(fp)\nwith open(path_cancer_0+\"percentile_75_cancer_0\", \"rb\") as fp:   \n    percentile_75_cancer_0 = pickle.load(fp)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:54:35.014305Z","iopub.execute_input":"2022-12-16T14:54:35.014842Z","iopub.status.idle":"2022-12-16T14:54:35.2875Z","shell.execute_reply.started":"2022-12-16T14:54:35.014789Z","shell.execute_reply":"2022-12-16T14:54:35.285874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batchsize=16\nmy_gen_cancer_1=image_batch_generator(train_data_cancer_1.path,batchsize=batchsize)\nbatch_count=int(len(train_data_cancer_1.path)/batchsize)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T14:35:53.159866Z","iopub.execute_input":"2022-12-14T14:35:53.160352Z","iopub.status.idle":"2022-12-14T14:35:53.172028Z","shell.execute_reply.started":"2022-12-14T14:35:53.160311Z","shell.execute_reply":"2022-12-14T14:35:53.170246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_data_cancer_1.path)/16","metadata":{"execution":{"iopub.status.busy":"2022-12-14T15:15:16.282183Z","iopub.execute_input":"2022-12-14T15:15:16.282652Z","iopub.status.idle":"2022-12-14T15:15:16.291089Z","shell.execute_reply.started":"2022-12-14T15:15:16.282609Z","shell.execute_reply":"2022-12-14T15:15:16.28917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(batch_count)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T14:36:04.726922Z","iopub.execute_input":"2022-12-14T14:36:04.727463Z","iopub.status.idle":"2022-12-14T14:36:04.735097Z","shell.execute_reply.started":"2022-12-14T14:36:04.727422Z","shell.execute_reply":"2022-12-14T14:36:04.733638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_list_cancer_1=[]\nmax_list_cancer_1=[]\nmean_cancer_1=[]\nmedian_cancer_1=[]\npercentile_25_cancer_1=[]\npercentile_75_cancer_1=[]\n\nfor i in range(0,batch_count):\n    print('calculating statistics for batch '+str(i))\n    batch=next(my_gen_cancer_1)\n    for j in range(0,batchsize):\n        min_list_cancer_1.append(np.min(batch[j]))\n        max_list_cancer_1.append(np.max(batch[j]))\n        mean_cancer_1.append(np.mean(batch[j]))\n        median_cancer_1.append(np.median(batch[j]))\n        percentile_25_cancer_1.append(np.percentile(batch[j],25))\n        percentile_75_cancer_1.append(np.percentile(batch[j],75))","metadata":{"execution":{"iopub.status.busy":"2022-12-14T14:36:35.158044Z","iopub.execute_input":"2022-12-14T14:36:35.159271Z","iopub.status.idle":"2022-12-14T14:37:53.075674Z","shell.execute_reply.started":"2022-12-14T14:36:35.159231Z","shell.execute_reply":"2022-12-14T14:37:53.074432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"min_cancer_1\", \"wb\") as fp:   #Pickling\n    pickle.dump(min_list_cancer_1, fp)\nwith open(\"max_cancer_1\", \"wb\") as fp:   #Pickling\n    pickle.dump(max_list_cancer_1, fp)\nwith open(\"mean_cancer_1\", \"wb\") as fp:   #Pickling\n    pickle.dump(mean_cancer_1, fp)\nwith open(\"median_cancer_1\", \"wb\") as fp:   #Pickling\n    pickle.dump(median_cancer_1, fp)\nwith open(\"percentile_25_cancer_1\", \"wb\") as fp:   #Pickling\n    pickle.dump(percentile_25_cancer_1, fp)\nwith open(\"percentile_75_cancer_1\", \"wb\") as fp:   #Pickling\n    pickle.dump(percentile_75_cancer_1, fp)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T14:41:52.339952Z","iopub.execute_input":"2022-12-14T14:41:52.340494Z","iopub.status.idle":"2022-12-14T14:41:52.37127Z","shell.execute_reply.started":"2022-12-14T14:41:52.340452Z","shell.execute_reply":"2022-12-14T14:41:52.369413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_cancer_1='/kaggle/input/image-stats-cancer-1/'","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:54:56.580761Z","iopub.execute_input":"2022-12-16T14:54:56.581507Z","iopub.status.idle":"2022-12-16T14:54:56.587645Z","shell.execute_reply.started":"2022-12-16T14:54:56.581405Z","shell.execute_reply":"2022-12-16T14:54:56.585983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(path_cancer_1+\"min_cancer_1\", \"rb\") as fp:   \n    min_cancer_1 = pickle.load(fp)\nwith open(path_cancer_1+\"max_cancer_1\", \"rb\") as fp:   \n    max_cancer_1 = pickle.load(fp)\nwith open(path_cancer_1+\"mean_cancer_1\", \"rb\") as fp:   \n    mean_cancer_1 = pickle.load(fp)\nwith open(path_cancer_1+\"median_cancer_1\", \"rb\") as fp:   \n    median_cancer_1 = pickle.load(fp)\nwith open(path_cancer_1+\"percentile_25_cancer_1\", \"rb\") as fp:   \n    percentile_25_cancer_1 = pickle.load(fp)\nwith open(path_cancer_1+\"percentile_75_cancer_1\", \"rb\") as fp:   \n    percentile_75_cancer_1 = pickle.load(fp)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:54:59.682021Z","iopub.execute_input":"2022-12-16T14:54:59.682413Z","iopub.status.idle":"2022-12-16T14:54:59.712525Z","shell.execute_reply.started":"2022-12-16T14:54:59.682381Z","shell.execute_reply":"2022-12-16T14:54:59.711583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cancer_0=pd.DataFrame()\ndf_cancer_0['min_cancer']=min_cancer_0\ndf_cancer_0['max_cancer']=max_cancer_0\ndf_cancer_0['mean_cancer']=mean_cancer_0\ndf_cancer_0['median_cancer']=median_cancer_0\ndf_cancer_0['percentile_25_cancer']=percentile_25_cancer_0\ndf_cancer_0['percentile_75_cancer']=percentile_75_cancer_0\ndf_cancer_0['cancer']=0","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:55:04.228173Z","iopub.execute_input":"2022-12-16T14:55:04.228628Z","iopub.status.idle":"2022-12-16T14:55:04.324215Z","shell.execute_reply.started":"2022-12-16T14:55:04.228594Z","shell.execute_reply":"2022-12-16T14:55:04.32292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cancer_1=pd.DataFrame()\ndf_cancer_1['min_cancer']=min_cancer_1\ndf_cancer_1['max_cancer']=max_cancer_1\ndf_cancer_1['mean_cancer']=mean_cancer_1\ndf_cancer_1['median_cancer']=median_cancer_1\ndf_cancer_1['percentile_25_cancer']=percentile_25_cancer_1\ndf_cancer_1['percentile_75_cancer']=percentile_75_cancer_1\ndf_cancer_1['cancer']=1","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:55:09.852307Z","iopub.execute_input":"2022-12-16T14:55:09.852827Z","iopub.status.idle":"2022-12-16T14:55:09.867923Z","shell.execute_reply.started":"2022-12-16T14:55:09.852786Z","shell.execute_reply":"2022-12-16T14:55:09.866581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_cancer_0.shape)\nprint(df_cancer_1.shape)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:55:15.366226Z","iopub.execute_input":"2022-12-16T14:55:15.366729Z","iopub.status.idle":"2022-12-16T14:55:15.373654Z","shell.execute_reply.started":"2022-12-16T14:55:15.366689Z","shell.execute_reply":"2022-12-16T14:55:15.372233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joint_df = pd.concat([df_cancer_0, df_cancer_1], axis=0, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:55:20.461133Z","iopub.execute_input":"2022-12-16T14:55:20.461588Z","iopub.status.idle":"2022-12-16T14:55:20.47028Z","shell.execute_reply.started":"2022-12-16T14:55:20.461552Z","shell.execute_reply":"2022-12-16T14:55:20.468522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joint_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:55:24.875974Z","iopub.execute_input":"2022-12-16T14:55:24.876418Z","iopub.status.idle":"2022-12-16T14:55:24.894258Z","shell.execute_reply.started":"2022-12-16T14:55:24.876384Z","shell.execute_reply":"2022-12-16T14:55:24.892631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joint_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-15T11:14:33.795995Z","iopub.execute_input":"2022-12-15T11:14:33.796736Z","iopub.status.idle":"2022-12-15T11:14:33.802827Z","shell.execute_reply.started":"2022-12-15T11:14:33.796697Z","shell.execute_reply":"2022-12-15T11:14:33.801926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [\n    \"min_cancer\", \"max_cancer\", \"mean_cancer\", \"median_cancer\", \n    \"percentile_25_cancer\", \"percentile_75_cancer\"]\n\nfrom statsmodels.regression.linear_model import OLS\nfrom statsmodels.tools.tools import add_constant\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\ntrain_subset = joint_df.copy()\nx = add_constant(train_subset[features])\n#x\nmodel = OLS(joint_df[\"cancer\"], x).fit()\n\npvalues = pd.DataFrame(model.pvalues)\npvalues.reset_index(inplace=True)\npvalues.rename(columns={0: \"pvalue\", \"index\": \"feature\"}, inplace=True)\npvalues.style.background_gradient(cmap='YlOrRd')","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:55:35.60089Z","iopub.execute_input":"2022-12-16T14:55:35.602159Z","iopub.status.idle":"2022-12-16T14:55:36.259059Z","shell.execute_reply.started":"2022-12-16T14:55:35.602108Z","shell.execute_reply":"2022-12-16T14:55:36.257675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The mean, median and 25th and 75th percentiles appear to be statistically significant in regard to cancer in images. Can build a simple classifier based on this. Mean most significant, next is 25th percentile, then median, then 75th percentile.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 12))\nplt.subplot(321)\nplt.title('Histogram of min values')\nplt.hist(min_cancer_0,label='cancer 0')\nplt.hist(min_cancer_1,label='cancer 1')\nplt.legend()\nplt.subplot(322)\nplt.title('Histogram of max values')\nplt.hist(max_cancer_0,label='cancer 0')\nplt.hist(max_cancer_1,label='cancer 1')\nplt.legend()\nplt.subplot(323)\nplt.title('Histogram of mean values')\nplt.hist(mean_cancer_0,label='cancer 0')\nplt.hist(mean_cancer_1,label='cancer 1')\nplt.legend()\nplt.subplot(324)\nplt.title('Histogram of median values')\nplt.hist(median_cancer_0,label='cancer 0')\nplt.hist(median_cancer_1,label='cancer 1')\nplt.legend()\nplt.subplot(325)\nplt.title('Histogram of 25th percentile values')\nplt.hist(percentile_25_cancer_0,label='cancer 0')\nplt.hist(percentile_25_cancer_1,label='cancer 1')\nplt.legend()\nplt.subplot(326)\nplt.title('Histogram of 75th percentile values')\nplt.hist(percentile_75_cancer_0,label='cancer 0')\nplt.hist(percentile_75_cancer_1,label='cancer 1')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:55:58.289818Z","iopub.execute_input":"2022-12-16T14:55:58.290243Z","iopub.status.idle":"2022-12-16T14:56:00.702849Z","shell.execute_reply.started":"2022-12-16T14:55:58.290211Z","shell.execute_reply":"2022-12-16T14:56:00.701526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Mean of minimum cancer 0')\nprint(np.mean(min_cancer_0))\nprint('Mean of minimum cancer 1')\nprint(np.mean(min_cancer_1))\nprint('Mean of maximum cancer 0')\nprint(np.mean(max_cancer_0))\nprint('Mean of maximum cancer 1')\nprint(np.mean(max_cancer_1))\nprint('Mean of mean cancer 0')\nprint(np.mean(mean_cancer_0))\nprint('Mean of mean cancer 1')\nprint(np.mean(mean_cancer_1))\nprint('Mean of median cancer 0')\nprint(np.mean(median_cancer_0))\nprint('Mean of median cancer 1')\nprint(np.mean(median_cancer_1))\nprint('Mean of 25th percentile cancer 0')\nprint(np.mean(percentile_25_cancer_0))\nprint('Mean of 25th percentile cancer 1')\nprint(np.mean(percentile_25_cancer_1))\nprint('Mean of 75th percentile cancer 0')\nprint(np.mean(percentile_75_cancer_0))\nprint('Mean of 75th percentile cancer 1')\nprint(np.mean(percentile_75_cancer_1))","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:56:06.636393Z","iopub.execute_input":"2022-12-16T14:56:06.636947Z","iopub.status.idle":"2022-12-16T14:56:06.672858Z","shell.execute_reply.started":"2022-12-16T14:56:06.636904Z","shell.execute_reply":"2022-12-16T14:56:06.671497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Shuffle training subset","metadata":{}},{"cell_type":"code","source":"train_subset_shuf=train_subset.sample(frac=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:56:15.357453Z","iopub.execute_input":"2022-12-16T14:56:15.357925Z","iopub.status.idle":"2022-12-16T14:56:15.371569Z","shell.execute_reply.started":"2022-12-16T14:56:15.357886Z","shell.execute_reply":"2022-12-16T14:56:15.370576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_subset_shuf.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:56:19.144537Z","iopub.execute_input":"2022-12-16T14:56:19.144972Z","iopub.status.idle":"2022-12-16T14:56:19.161187Z","shell.execute_reply.started":"2022-12-16T14:56:19.144938Z","shell.execute_reply":"2022-12-16T14:56:19.160339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_subset_shuf.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-15T13:36:46.817146Z","iopub.execute_input":"2022-12-15T13:36:46.817924Z","iopub.status.idle":"2022-12-15T13:36:46.825989Z","shell.execute_reply.started":"2022-12-15T13:36:46.817893Z","shell.execute_reply":"2022-12-15T13:36:46.825211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Split training set into a smaller training and validation set","metadata":{}},{"cell_type":"code","source":"fraction=0.8\ntraining_smaller=train_subset_shuf.iloc[0:int(fraction*train_subset_shuf.shape[0])]\nvalidation_smaller=train_subset_shuf.iloc[int(fraction*train_subset_shuf.shape[0]):int(train_subset_shuf.shape[0])]","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:56:30.771812Z","iopub.execute_input":"2022-12-16T14:56:30.772254Z","iopub.status.idle":"2022-12-16T14:56:30.779393Z","shell.execute_reply.started":"2022-12-16T14:56:30.772213Z","shell.execute_reply":"2022-12-16T14:56:30.778326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(training_smaller.shape)\nprint(validation_smaller.shape)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:56:35.259626Z","iopub.execute_input":"2022-12-16T14:56:35.260025Z","iopub.status.idle":"2022-12-16T14:56:35.267448Z","shell.execute_reply.started":"2022-12-16T14:56:35.259995Z","shell.execute_reply":"2022-12-16T14:56:35.265507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_smaller_cancer_0=training_smaller[training_smaller.cancer==0]\ntraining_smaller_cancer_1=training_smaller[training_smaller.cancer==1]","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:57:16.307652Z","iopub.execute_input":"2022-12-16T14:57:16.308127Z","iopub.status.idle":"2022-12-16T14:57:16.320362Z","shell.execute_reply.started":"2022-12-16T14:57:16.308088Z","shell.execute_reply":"2022-12-16T14:57:16.318776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(training_smaller_cancer_0.head())\nprint(training_smaller_cancer_1.head())","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:57:24.35133Z","iopub.execute_input":"2022-12-16T14:57:24.351888Z","iopub.status.idle":"2022-12-16T14:57:24.369389Z","shell.execute_reply.started":"2022-12-16T14:57:24.35185Z","shell.execute_reply":"2022-12-16T14:57:24.368339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculate thresholds based on mean values of the statistically significant variables","metadata":{}},{"cell_type":"code","source":"print(np.mean(training_smaller_cancer_0.mean_cancer))\nprint(np.mean(training_smaller_cancer_1.mean_cancer))","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:11:09.891643Z","iopub.execute_input":"2022-12-16T15:11:09.892232Z","iopub.status.idle":"2022-12-16T15:11:09.90268Z","shell.execute_reply.started":"2022-12-16T15:11:09.892177Z","shell.execute_reply":"2022-12-16T15:11:09.900863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thres_mean=(np.mean(training_smaller_cancer_0.mean_cancer)+\n            np.mean(training_smaller_cancer_1.mean_cancer))/2\nthres_median=(np.mean(training_smaller_cancer_0.median_cancer)+\n            np.mean(training_smaller_cancer_1.median_cancer))/2\nthres_25p=(np.mean(training_smaller_cancer_0.percentile_25_cancer)+\n            np.mean(training_smaller_cancer_1.percentile_25_cancer))/2\nthres_75p=(np.mean(training_smaller_cancer_0.percentile_75_cancer)+\n            np.mean(training_smaller_cancer_1.percentile_75_cancer))/2","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:57:33.275612Z","iopub.execute_input":"2022-12-16T14:57:33.276674Z","iopub.status.idle":"2022-12-16T14:57:33.286137Z","shell.execute_reply.started":"2022-12-16T14:57:33.276626Z","shell.execute_reply":"2022-12-16T14:57:33.285198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(thres_mean)\nprint(thres_median)\nprint(thres_25p)\nprint(thres_75p)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T14:57:42.30342Z","iopub.execute_input":"2022-12-16T14:57:42.303993Z","iopub.status.idle":"2022-12-16T14:57:42.311552Z","shell.execute_reply.started":"2022-12-16T14:57:42.303953Z","shell.execute_reply":"2022-12-16T14:57:42.310305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller.mean_cancer","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:14:52.357075Z","iopub.execute_input":"2022-12-16T15:14:52.35753Z","iopub.status.idle":"2022-12-16T15:14:52.369002Z","shell.execute_reply.started":"2022-12-16T15:14:52.357493Z","shell.execute_reply":"2022-12-16T15:14:52.367543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller['test_mean']=validation_smaller.mean_cancer<thres_mean\nvalidation_smaller['test_median']=validation_smaller.median_cancer<thres_median\nvalidation_smaller['test_25p']=validation_smaller.percentile_25_cancer>thres_25p\nvalidation_smaller['test_75p']=validation_smaller.percentile_75_cancer<thres_75p","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:32:11.982184Z","iopub.execute_input":"2022-12-16T15:32:11.98274Z","iopub.status.idle":"2022-12-16T15:32:11.997631Z","shell.execute_reply.started":"2022-12-16T15:32:11.982696Z","shell.execute_reply":"2022-12-16T15:32:11.996007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:32:17.836016Z","iopub.execute_input":"2022-12-16T15:32:17.836509Z","iopub.status.idle":"2022-12-16T15:32:17.868378Z","shell.execute_reply.started":"2022-12-16T15:32:17.836454Z","shell.execute_reply":"2022-12-16T15:32:17.867552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_conditions_met=validation_smaller.test_mean*validation_smaller.test_median*validation_smaller.test_25p*validation_smaller.test_75p","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:35:45.542292Z","iopub.execute_input":"2022-12-16T15:35:45.542733Z","iopub.status.idle":"2022-12-16T15:35:45.550746Z","shell.execute_reply.started":"2022-12-16T15:35:45.542688Z","shell.execute_reply":"2022-12-16T15:35:45.549346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_conditions_met","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:45:32.805941Z","iopub.execute_input":"2022-12-16T15:45:32.80638Z","iopub.status.idle":"2022-12-16T15:45:32.818201Z","shell.execute_reply.started":"2022-12-16T15:45:32.806346Z","shell.execute_reply":"2022-12-16T15:45:32.816574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(all_conditions_met)/validation_smaller.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:36:29.824865Z","iopub.execute_input":"2022-12-16T15:36:29.825349Z","iopub.status.idle":"2022-12-16T15:36:29.834831Z","shell.execute_reply.started":"2022-12-16T15:36:29.825312Z","shell.execute_reply":"2022-12-16T15:36:29.833579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller['all_conditions_met']=all_conditions_met","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:37:48.095815Z","iopub.execute_input":"2022-12-16T15:37:48.096391Z","iopub.status.idle":"2022-12-16T15:37:48.104401Z","shell.execute_reply.started":"2022-12-16T15:37:48.096347Z","shell.execute_reply":"2022-12-16T15:37:48.103259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:39:09.501451Z","iopub.execute_input":"2022-12-16T15:39:09.502414Z","iopub.status.idle":"2022-12-16T15:39:09.531593Z","shell.execute_reply.started":"2022-12-16T15:39:09.502361Z","shell.execute_reply":"2022-12-16T15:39:09.530092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller[validation_smaller['cancer']==1]['all_conditions_met']","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:41:37.390049Z","iopub.execute_input":"2022-12-16T15:41:37.390546Z","iopub.status.idle":"2022-12-16T15:41:37.402226Z","shell.execute_reply.started":"2022-12-16T15:41:37.390495Z","shell.execute_reply":"2022-12-16T15:41:37.400935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.sum(validation_smaller[validation_smaller['cancer']==1]['all_conditions_met']))","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:42:17.897412Z","iopub.execute_input":"2022-12-16T15:42:17.897948Z","iopub.status.idle":"2022-12-16T15:42:17.907678Z","shell.execute_reply.started":"2022-12-16T15:42:17.897908Z","shell.execute_reply":"2022-12-16T15:42:17.905839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller[validation_smaller['all_conditions_met']=='True']","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:44:31.251135Z","iopub.execute_input":"2022-12-16T15:44:31.251568Z","iopub.status.idle":"2022-12-16T15:44:31.266524Z","shell.execute_reply.started":"2022-12-16T15:44:31.25153Z","shell.execute_reply":"2022-12-16T15:44:31.265343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller[validation_smaller['all_conditions_met']=='True']['cancer']","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:44:05.294821Z","iopub.execute_input":"2022-12-16T15:44:05.295392Z","iopub.status.idle":"2022-12-16T15:44:05.306419Z","shell.execute_reply.started":"2022-12-16T15:44:05.295349Z","shell.execute_reply":"2022-12-16T15:44:05.305047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.sum(validation_smaller.mean_cancer<thres_mean))\nprint(np.sum(validation_smaller.mean_cancer>thres_mean))","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:43:25.431126Z","iopub.execute_input":"2022-12-16T15:43:25.431606Z","iopub.status.idle":"2022-12-16T15:43:25.440153Z","shell.execute_reply.started":"2022-12-16T15:43:25.431569Z","shell.execute_reply":"2022-12-16T15:43:25.438097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As mentioned before, in terms of p-values, the mean is most significant, next is 25th percentile, then median, then 75th percentile. For cancer to be likely, the mean, median and 75th percentile all have to be below the threshold, whereas the 25th percentle has to be above the threshold.Therefore, can assign weights based on this. Choose weight of 5 for mean, 4 for 25th percentile, 2 for median and 1 for 75th percentile  ","metadata":{}},{"cell_type":"code","source":"weight_mean=5\nweight_percentile_25=4\nweight_median=2\nweight_percentile_75=1","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:01:51.944779Z","iopub.execute_input":"2022-12-16T15:01:51.945267Z","iopub.status.idle":"2022-12-16T15:01:51.951599Z","shell.execute_reply.started":"2022-12-16T15:01:51.945229Z","shell.execute_reply":"2022-12-16T15:01:51.950227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller.columns\nlikely_cancer_mean=weight_mean*(thres_mean-validation_smaller.mean_cancer)\nlikely_cancer_median=weight_median*(thres_median-validation_smaller.median_cancer)\nlikely_cancer_percentile_25=-weight_percentile_25*(thres_25p-validation_smaller.percentile_25_cancer)\nlikely_cancer_percentile_75=weight_percentile_75*(thres_75p-validation_smaller.percentile_75_cancer)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:02:03.790279Z","iopub.execute_input":"2022-12-16T15:02:03.791794Z","iopub.status.idle":"2022-12-16T15:02:03.803069Z","shell.execute_reply.started":"2022-12-16T15:02:03.791745Z","shell.execute_reply":"2022-12-16T15:02:03.801784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(likely_cancer_mean)\nprint(likely_cancer_median)\nprint(likely_cancer_percentile_25)\nprint(likely_cancer_percentile_75)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:02:07.426685Z","iopub.execute_input":"2022-12-16T15:02:07.427161Z","iopub.status.idle":"2022-12-16T15:02:07.440562Z","shell.execute_reply.started":"2022-12-16T15:02:07.427124Z","shell.execute_reply":"2022-12-16T15:02:07.439017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller['likely_cancer_mean']=likely_cancer_mean\nvalidation_smaller['likely_cancer_median']=likely_cancer_median\nvalidation_smaller['likely_cancer_percentile_25']=likely_cancer_percentile_25\nvalidation_smaller['likely_cancer_percentile_75']=likely_cancer_percentile_75","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:02:14.309592Z","iopub.execute_input":"2022-12-16T15:02:14.31087Z","iopub.status.idle":"2022-12-16T15:02:14.322269Z","shell.execute_reply.started":"2022-12-16T15:02:14.310821Z","shell.execute_reply":"2022-12-16T15:02:14.320947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:02:28.752743Z","iopub.execute_input":"2022-12-16T15:02:28.75321Z","iopub.status.idle":"2022-12-16T15:02:28.780933Z","shell.execute_reply.started":"2022-12-16T15:02:28.753174Z","shell.execute_reply":"2022-12-16T15:02:28.779365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller_0=validation_smaller[validation_smaller.cancer==0]\nvalidation_smaller_1=validation_smaller[validation_smaller.cancer==1]","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:04:30.785186Z","iopub.execute_input":"2022-12-16T15:04:30.786541Z","iopub.status.idle":"2022-12-16T15:04:30.796363Z","shell.execute_reply.started":"2022-12-16T15:04:30.786456Z","shell.execute_reply":"2022-12-16T15:04:30.794701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller_1\nprint(np.max(validation_smaller_1.likely_cancer_mean))\nprint(np.max(validation_smaller_0.likely_cancer_mean))","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:06:46.559716Z","iopub.execute_input":"2022-12-16T15:06:46.560128Z","iopub.status.idle":"2022-12-16T15:06:46.568435Z","shell.execute_reply.started":"2022-12-16T15:06:46.560095Z","shell.execute_reply":"2022-12-16T15:06:46.56663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"likely_cancer_pred=likely_cancer_mean+likely_cancer_median+likely_cancer_percentile_25+likely_cancer_percentile_75","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:12:50.10746Z","iopub.execute_input":"2022-12-15T14:12:50.107834Z","iopub.status.idle":"2022-12-15T14:12:50.11359Z","shell.execute_reply.started":"2022-12-15T14:12:50.107804Z","shell.execute_reply":"2022-12-15T14:12:50.112482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint(likely_cancer_pred)\nprint(likely_cancer_pred/4)\naveraged_pred=likely_cancer_pred/4","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:14:33.604115Z","iopub.execute_input":"2022-12-15T14:14:33.604487Z","iopub.status.idle":"2022-12-15T14:14:33.612149Z","shell.execute_reply.started":"2022-12-15T14:14:33.604459Z","shell.execute_reply":"2022-12-15T14:14:33.611134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.min(averaged_pred))\nprint(np.max(averaged_pred))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:15:40.637819Z","iopub.execute_input":"2022-12-15T14:15:40.638173Z","iopub.status.idle":"2022-12-15T14:15:40.643687Z","shell.execute_reply.started":"2022-12-15T14:15:40.638142Z","shell.execute_reply":"2022-12-15T14:15:40.642698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(averaged_pred)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:20:57.86426Z","iopub.execute_input":"2022-12-15T14:20:57.864662Z","iopub.status.idle":"2022-12-15T14:20:58.025596Z","shell.execute_reply.started":"2022-12-15T14:20:57.86463Z","shell.execute_reply":"2022-12-15T14:20:58.024498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Normalise","metadata":{}},{"cell_type":"code","source":"averaged_pred_norm=(averaged_pred-np.min(averaged_pred))/(np.max(averaged_pred)-np.min(averaged_pred))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:19:25.073227Z","iopub.execute_input":"2022-12-15T14:19:25.073605Z","iopub.status.idle":"2022-12-15T14:19:25.081027Z","shell.execute_reply.started":"2022-12-15T14:19:25.073577Z","shell.execute_reply":"2022-12-15T14:19:25.07983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.min(averaged_pred_norm))\nprint(np.max(averaged_pred_norm))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:19:35.543362Z","iopub.execute_input":"2022-12-15T14:19:35.543718Z","iopub.status.idle":"2022-12-15T14:19:35.55007Z","shell.execute_reply.started":"2022-12-15T14:19:35.543692Z","shell.execute_reply":"2022-12-15T14:19:35.548719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(averaged_pred_norm,bins=30)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:22:22.847298Z","iopub.execute_input":"2022-12-15T14:22:22.847703Z","iopub.status.idle":"2022-12-15T14:22:23.042162Z","shell.execute_reply.started":"2022-12-15T14:22:22.847674Z","shell.execute_reply":"2022-12-15T14:22:23.041313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.sum(validation_smaller.cancer==1))\nprint(np.sum(validation_smaller.cancer==0))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:27:00.565002Z","iopub.execute_input":"2022-12-15T14:27:00.565358Z","iopub.status.idle":"2022-12-15T14:27:00.571784Z","shell.execute_reply.started":"2022-12-15T14:27:00.56533Z","shell.execute_reply":"2022-12-15T14:27:00.570677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.sum(validation_smaller.cancer==1)/(np.sum(validation_smaller.cancer==1)+np.sum(validation_smaller.cancer==0)))\nprint(np.sum(validation_smaller.cancer==0)/(np.sum(validation_smaller.cancer==1)+np.sum(validation_smaller.cancer==0)))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:28:39.19062Z","iopub.execute_input":"2022-12-15T14:28:39.191025Z","iopub.status.idle":"2022-12-15T14:28:39.199892Z","shell.execute_reply.started":"2022-12-15T14:28:39.190994Z","shell.execute_reply":"2022-12-15T14:28:39.198461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"98% of the data has a cancer=0 classification. Therefore choose a cut-off in the histogram above to reflect this","metadata":{}},{"cell_type":"code","source":"averaged_pred_norm_cancer_0=averaged_pred_norm[averaged_pred_norm<=0.9693]\naveraged_pred_norm_cancer_1=averaged_pred_norm[averaged_pred_norm>0.9693]","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:35:38.466549Z","iopub.execute_input":"2022-12-15T14:35:38.466927Z","iopub.status.idle":"2022-12-15T14:35:38.474208Z","shell.execute_reply.started":"2022-12-15T14:35:38.466895Z","shell.execute_reply":"2022-12-15T14:35:38.472966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(averaged_pred_norm_cancer_0))\nprint(len(averaged_pred_norm_cancer_1))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:35:49.704666Z","iopub.execute_input":"2022-12-15T14:35:49.70505Z","iopub.status.idle":"2022-12-15T14:35:49.710979Z","shell.execute_reply.started":"2022-12-15T14:35:49.705019Z","shell.execute_reply":"2022-12-15T14:35:49.709596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cancer_pred_total=np.repeat(0,len(averaged_pred_norm))\n#print(type(cancer_pred_total))\n#c = (cancer_pred_total < 0.9693).astype(int)\n#if (averaged_pred_norm<=0.9693):\n#    cancer_pred_total=0\naveraged_pred_norm[averaged_pred_norm > 0.9693] = 1\nprint(averaged_pred_norm)\nlen(np.where(averaged_pred_norm==1)[0])","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:48:41.341854Z","iopub.execute_input":"2022-12-15T14:48:41.342235Z","iopub.status.idle":"2022-12-15T14:48:41.354538Z","shell.execute_reply.started":"2022-12-15T14:48:41.342206Z","shell.execute_reply":"2022-12-15T14:48:41.352993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller['preds']=averaged_pred_norm","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:50:17.874808Z","iopub.execute_input":"2022-12-15T14:50:17.875468Z","iopub.status.idle":"2022-12-15T14:50:17.884433Z","shell.execute_reply.started":"2022-12-15T14:50:17.875414Z","shell.execute_reply":"2022-12-15T14:50:17.883134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_smaller.columns","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:50:23.022866Z","iopub.execute_input":"2022-12-15T14:50:23.023263Z","iopub.status.idle":"2022-12-15T14:50:23.031297Z","shell.execute_reply.started":"2022-12-15T14:50:23.023231Z","shell.execute_reply":"2022-12-15T14:50:23.029886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean((validation_smaller.cancer-validation_smaller.preds)**2)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:51:41.591691Z","iopub.execute_input":"2022-12-15T14:51:41.592124Z","iopub.status.idle":"2022-12-15T14:51:41.599342Z","shell.execute_reply.started":"2022-12-15T14:51:41.592091Z","shell.execute_reply":"2022-12-15T14:51:41.598426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Comparison_cancer_1_true = validation_smaller[validation_smaller.cancer==1]\nComparison_cancer_1_pred = validation_smaller[validation_smaller.preds==1]","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:56:16.631174Z","iopub.execute_input":"2022-12-15T14:56:16.631816Z","iopub.status.idle":"2022-12-15T14:56:16.638433Z","shell.execute_reply.started":"2022-12-15T14:56:16.631753Z","shell.execute_reply":"2022-12-15T14:56:16.637335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(Comparison_cancer_1_true[0:50])\nprint(Comparison_cancer_1_pred[0:50])","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:57:03.506979Z","iopub.execute_input":"2022-12-15T14:57:03.507293Z","iopub.status.idle":"2022-12-15T14:57:03.534225Z","shell.execute_reply.started":"2022-12-15T14:57:03.507267Z","shell.execute_reply":"2022-12-15T14:57:03.533008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thresholds based on entire training set","metadata":{}},{"cell_type":"code","source":"thres_mean_all=(np.mean(mean_cancer_0)+np.mean(mean_cancer_1))/2\nthres_median_all=(np.mean(median_cancer_0)+np.mean(median_cancer_1))/2\nthres_25p_all=(np.mean(percentile_25_cancer_0)+np.mean(percentile_25_cancer_1))/2\nthres_75p_all=(np.mean(percentile_75_cancer_0)+np.mean(percentile_75_cancer_1))/2","metadata":{"execution":{"iopub.status.busy":"2022-12-15T13:49:45.667701Z","iopub.execute_input":"2022-12-15T13:49:45.668057Z","iopub.status.idle":"2022-12-15T13:49:45.686059Z","shell.execute_reply.started":"2022-12-15T13:49:45.668029Z","shell.execute_reply":"2022-12-15T13:49:45.68523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(thres_mean_all)\nprint(thres_median_all)\nprint(thres_25p_all)\nprint(thres_75p_all)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T13:49:48.457845Z","iopub.execute_input":"2022-12-15T13:49:48.458631Z","iopub.status.idle":"2022-12-15T13:49:48.462718Z","shell.execute_reply.started":"2022-12-15T13:49:48.458599Z","shell.execute_reply":"2022-12-15T13:49:48.462079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read in test images","metadata":{}},{"cell_type":"code","source":"test_images = sorted(glob.glob(\"../input/rsna-breast-cancer-detection/test_images/10008/*\"))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:58:52.441815Z","iopub.execute_input":"2022-12-15T14:58:52.442171Z","iopub.status.idle":"2022-12-15T14:58:52.45046Z","shell.execute_reply.started":"2022-12-15T14:58:52.442143Z","shell.execute_reply":"2022-12-15T14:58:52.449357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images","metadata":{"execution":{"iopub.status.busy":"2022-12-15T14:58:55.594967Z","iopub.execute_input":"2022-12-15T14:58:55.595316Z","iopub.status.idle":"2022-12-15T14:58:55.602184Z","shell.execute_reply.started":"2022-12-15T14:58:55.595274Z","shell.execute_reply":"2022-12-15T14:58:55.601101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Process test images using steps in https://www.kaggle.com/code/radek1/how-to-process-dicom-images-to-pngs","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf test_images_processed_{512}","metadata":{"execution":{"iopub.status.busy":"2022-12-15T12:12:28.710038Z","iopub.execute_input":"2022-12-15T12:12:28.710545Z","iopub.status.idle":"2022-12-15T12:12:29.808638Z","shell.execute_reply.started":"2022-12-15T12:12:28.710506Z","shell.execute_reply":"2022-12-15T12:12:29.806683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nfrom pathlib import Path\nimport numpy as np\nfrom PIL import Image\n\nRESIZE_TO = (1024, 1024)\n!rm -rf test_images_processed_{RESIZE_TO[0]}\n!mkdir test_images_processed_{RESIZE_TO[0]}","metadata":{"execution":{"iopub.status.busy":"2022-12-15T12:11:33.390723Z","iopub.execute_input":"2022-12-15T12:11:33.391545Z","iopub.status.idle":"2022-12-15T12:11:35.609636Z","shell.execute_reply.started":"2022-12-15T12:11:33.391488Z","shell.execute_reply":"2022-12-15T12:11:35.608202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicom_file_to_ary(path):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2022-12-15T12:12:49.795437Z","iopub.execute_input":"2022-12-15T12:12:49.795922Z","iopub.status.idle":"2022-12-15T12:12:49.804796Z","shell.execute_reply.started":"2022-12-15T12:12:49.795882Z","shell.execute_reply":"2022-12-15T12:12:49.803478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path=path+'test_images/10008/*'","metadata":{"execution":{"iopub.status.busy":"2022-12-15T12:12:52.326217Z","iopub.execute_input":"2022-12-15T12:12:52.326679Z","iopub.status.idle":"2022-12-15T12:12:52.333451Z","shell.execute_reply.started":"2022-12-15T12:12:52.326644Z","shell.execute_reply":"2022-12-15T12:12:52.332236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"first_test_img=sorted(glob.glob(test_path))[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-15T12:12:59.711631Z","iopub.execute_input":"2022-12-15T12:12:59.712169Z","iopub.status.idle":"2022-12-15T12:12:59.720405Z","shell.execute_reply.started":"2022-12-15T12:12:59.712132Z","shell.execute_reply":"2022-12-15T12:12:59.719093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img=dicom_file_to_ary(first_test_img)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T12:13:04.174152Z","iopub.execute_input":"2022-12-15T12:13:04.17496Z","iopub.status.idle":"2022-12-15T12:13:04.92609Z","shell.execute_reply.started":"2022-12-15T12:13:04.174902Z","shell.execute_reply":"2022-12-15T12:13:04.924771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-15T12:13:06.9052Z","iopub.execute_input":"2022-12-15T12:13:06.90601Z","iopub.status.idle":"2022-12-15T12:13:06.913044Z","shell.execute_reply.started":"2022-12-15T12:13:06.905964Z","shell.execute_reply":"2022-12-15T12:13:06.912222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"directories = list(Path(path+'test_images').iterdir())","metadata":{"execution":{"iopub.status.busy":"2022-12-15T12:16:59.412364Z","iopub.execute_input":"2022-12-15T12:16:59.413315Z","iopub.status.idle":"2022-12-15T12:16:59.422776Z","shell.execute_reply.started":"2022-12-15T12:16:59.413266Z","shell.execute_reply":"2022-12-15T12:16:59.421662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"directories","metadata":{"execution":{"iopub.status.busy":"2022-12-15T12:17:08.394999Z","iopub.execute_input":"2022-12-15T12:17:08.396052Z","iopub.status.idle":"2022-12-15T12:17:08.405036Z","shell.execute_reply.started":"2022-12-15T12:17:08.395993Z","shell.execute_reply":"2022-12-15T12:17:08.403665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/tanlikesmath/brain-tumor-radiogenomic-classification-eda/notebook\ndef process_directory(directory_path):\n    parent_directory = str(directory_path).split('/')[-1]\n    !mkdir -p test_images_processed_{RESIZE_TO[0]}/{parent_directory}\n    for image_path in directory_path.iterdir():\n        processed_ary = dicom_file_to_ary(image_path)\n        im = Image.fromarray(processed_ary).resize(RESIZE_TO)\n        im.save(f'test_images_processed_{RESIZE_TO[0]}/{parent_directory}/{image_path.stem}.png')\n        \nimport multiprocessing as mp\n\nwith mp.Pool(64) as p:\n    p.map(process_directory, directories)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T12:18:04.018717Z","iopub.execute_input":"2022-12-15T12:18:04.019412Z","iopub.status.idle":"2022-12-15T12:18:10.385313Z","shell.execute_reply.started":"2022-12-15T12:18:04.019375Z","shell.execute_reply":"2022-12-15T12:18:10.383512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(path)\ntest_df = pd.read_csv(path+'test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-15T11:35:21.515062Z","iopub.execute_input":"2022-12-15T11:35:21.515856Z","iopub.status.idle":"2022-12-15T11:35:21.542923Z","shell.execute_reply.started":"2022-12-15T11:35:21.515813Z","shell.execute_reply":"2022-12-15T11:35:21.541663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-15T11:35:31.982915Z","iopub.execute_input":"2022-12-15T11:35:31.983929Z","iopub.status.idle":"2022-12-15T11:35:31.99887Z","shell.execute_reply.started":"2022-12-15T11:35:31.98388Z","shell.execute_reply":"2022-12-15T11:35:31.997497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=[0.25,0.25,0.25,0.25]","metadata":{"execution":{"iopub.status.busy":"2022-12-15T11:37:35.098053Z","iopub.execute_input":"2022-12-15T11:37:35.098524Z","iopub.status.idle":"2022-12-15T11:37:35.10473Z","shell.execute_reply.started":"2022-12-15T11:37:35.098491Z","shell.execute_reply":"2022-12-15T11:37:35.103239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.DataFrame({'prediction_id':test_df.prediction_id,\n                        'cancer':preds})\nsub_df = pd.read_csv(path+'sample_submission.csv')\nprint(pred_df)\nprint(sub_df)\ndel sub_df['cancer']\nsub_df = sub_df.merge(pred_df, on='prediction_id', how='left')\nsub_df = sub_df.groupby('prediction_id')['cancer'].max().reset_index()  # merge duplicate prediction_id\n#sub_df['cancer'] = (sub_df.cancer>CFG.thr).astype('float32') # thresholding\n#sub_df.to_csv('submission.csv',index=False)\n#sub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-15T11:38:59.833224Z","iopub.execute_input":"2022-12-15T11:38:59.834078Z","iopub.status.idle":"2022-12-15T11:38:59.858014Z","shell.execute_reply.started":"2022-12-15T11:38:59.834038Z","shell.execute_reply":"2022-12-15T11:38:59.856856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cancer_0_imgs_all=[]\n\nfor i in range(0,len(cancer_0_idx)):\n    img = cv2.imread(train_data_cancer_0['path'][cancer_0_idx[i]], cv2.IMREAD_GRAYSCALE)\n    cancer_0_imgs_all.append(img)","metadata":{"execution":{"iopub.status.busy":"2022-12-13T13:51:54.736946Z","iopub.execute_input":"2022-12-13T13:51:54.737414Z","iopub.status.idle":"2022-12-13T13:58:39.463985Z","shell.execute_reply.started":"2022-12-13T13:51:54.737379Z","shell.execute_reply":"2022-12-13T13:58:39.462684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(cancer_0_imgs_all))\nprint(len(cancer_0_idx))","metadata":{"execution":{"iopub.status.busy":"2022-12-13T14:00:05.024868Z","iopub.execute_input":"2022-12-13T14:00:05.025327Z","iopub.status.idle":"2022-12-13T14:00:05.032246Z","shell.execute_reply.started":"2022-12-13T14:00:05.025292Z","shell.execute_reply":"2022-12-13T14:00:05.030893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_cancer_0=[]\nmax_cancer_0=[]\nmean_cancer_0=[]\nmedian_cancer_0=[]\npercentile_25_cancer_0=[]\npercentile_75_cancer_0=[]\n\nfor i in range(0,len(cancer_0_imgs_all)):\n    min_cancer_0.append(np.min(cancer_0_imgs_all[i]))\n    max_cancer_0.append(np.max(cancer_0_imgs_all[i]))\n    mean_cancer_0.append(np.mean(cancer_0_imgs_all[i]))\n    median_cancer_0.append(np.median(cancer_0_imgs_all[i]))\n    percentile_25_cancer_0.append(np.percentile(cancer_0_imgs_all[i],25))\n    percentile_75_cancer_0.append(np.percentile(cancer_0_imgs_all[i],75))","metadata":{"execution":{"iopub.status.busy":"2022-12-13T14:03:55.702653Z","iopub.execute_input":"2022-12-13T14:03:55.703076Z","iopub.status.idle":"2022-12-13T14:08:46.902649Z","shell.execute_reply.started":"2022-12-13T14:03:55.703021Z","shell.execute_reply":"2022-12-13T14:08:46.901064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Collected a list of images that is under half of the total available for class 0.","metadata":{}},{"cell_type":"code","source":"print('Mean of minimum cancer 0')\nprint(np.mean(min_cancer_0))\nprint('Mean of minimum cancer 1')\nprint(np.mean(min_cancer_1))\nprint('Mean of maximum cancer 0')\nprint(np.mean(max_cancer_0))\nprint('Mean of maximum cancer 1')\nprint(np.mean(max_cancer_1))\nprint('Mean of mean cancer 0')\nprint(np.mean(mean_cancer_0))\nprint('Mean of mean cancer 1')\nprint(np.mean(mean_cancer_1))\nprint('Mean of median cancer 0')\nprint(np.median(mean_cancer_0))\nprint('Mean of median cancer 1')\nprint(np.median(mean_cancer_1))","metadata":{"execution":{"iopub.status.busy":"2022-12-13T14:09:17.558932Z","iopub.execute_input":"2022-12-13T14:09:17.559352Z","iopub.status.idle":"2022-12-13T14:09:17.5821Z","shell.execute_reply.started":"2022-12-13T14:09:17.55932Z","shell.execute_reply":"2022-12-13T14:09:17.580606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-13T13:59:15.343543Z","iopub.execute_input":"2022-12-13T13:59:15.343969Z","iopub.status.idle":"2022-12-13T13:59:15.738496Z","shell.execute_reply.started":"2022-12-13T13:59:15.343934Z","shell.execute_reply":"2022-12-13T13:59:15.736883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example = path+'/train_images/10006/1459541791.dcm'","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:39:30.278081Z","iopub.execute_input":"2022-12-07T15:39:30.278478Z","iopub.status.idle":"2022-12-07T15:39:30.283896Z","shell.execute_reply.started":"2022-12-07T15:39:30.278428Z","shell.execute_reply":"2022-12-07T15:39:30.282613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# source: https://www.kaggle.com/code/allunia/rsna-csf-cervical-spine-fracture-eda/notebook\ndef rescale_img_to_hu(dcm_ds):\n    \"\"\"Rescales the image to Hounsfield unit.\"\"\"\n    data = dcm_ds.pixel_array\n    if dcm_ds.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    return data * dcm_ds.RescaleSlope + dcm_ds.RescaleIntercept","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:40:36.743196Z","iopub.execute_input":"2022-12-07T15:40:36.744413Z","iopub.status.idle":"2022-12-07T15:40:36.750745Z","shell.execute_reply.started":"2022-12-07T15:40:36.74437Z","shell.execute_reply":"2022-12-07T15:40:36.749809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_images_for_patient(patient_id):\n    patient_dir = os.path.join('../input/rsna-breast-cancer-detection/train_images', str(patient_id))\n    num_images = len(glob.glob(f\"{patient_dir}/*\"))\n    print(f\"Number of images for patient: {num_images}\")\n    fig, axs = plt.subplots(2, 2, figsize=(24,15))\n    axs = axs.flatten()\n    for i, img_path in enumerate(list(Path(patient_dir).iterdir())):\n        ds = pydicom.dcmread(img_path)\n        axs[i].imshow(rescale_img_to_hu(ds), cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:40:41.741646Z","iopub.execute_input":"2022-12-07T15:40:41.742433Z","iopub.status.idle":"2022-12-07T15:40:41.750785Z","shell.execute_reply.started":"2022-12-07T15:40:41.742391Z","shell.execute_reply":"2022-12-07T15:40:41.74936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"End of copied part from https://www.kaggle.com/code/radek1/eda-training-a-fast-ai-model-submission","metadata":{}},{"cell_type":"markdown","source":"Looking at the training data for both cancer and no-cancer classification","metadata":{}},{"cell_type":"code","source":"train_data_cancer=train_data[train_data.cancer==1]\ntrain_data_no_cancer=train_data[train_data.cancer==0]","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:41:16.209822Z","iopub.execute_input":"2022-12-07T15:41:16.210223Z","iopub.status.idle":"2022-12-07T15:41:16.225918Z","shell.execute_reply.started":"2022-12-07T15:41:16.210189Z","shell.execute_reply":"2022-12-07T15:41:16.224535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data_cancer.shape)\nprint(train_data_cancer)\nprint(train_data_no_cancer.shape)\nprint(train_data_no_cancer)","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:41:19.169224Z","iopub.execute_input":"2022-12-07T15:41:19.169638Z","iopub.status.idle":"2022-12-07T15:41:19.19901Z","shell.execute_reply.started":"2022-12-07T15:41:19.169602Z","shell.execute_reply":"2022-12-07T15:41:19.197949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id_cancer=train_data_cancer.patient_id.unique()\npatient_id_no_cancer=train_data_no_cancer.patient_id.unique()\n","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:41:23.618315Z","iopub.execute_input":"2022-12-07T15:41:23.618737Z","iopub.status.idle":"2022-12-07T15:41:23.628091Z","shell.execute_reply.started":"2022-12-07T15:41:23.618705Z","shell.execute_reply":"2022-12-07T15:41:23.626774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def number_of_img_for_patient(patient_id):\n    patient_dir = os.path.join('../input/rsna-breast-cancer-detection/train_images', str(patient_id))\n    num_images = len(glob.glob(f\"{patient_dir}/*\"))\n    return num_images","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:41:28.108164Z","iopub.execute_input":"2022-12-07T15:41:28.108573Z","iopub.status.idle":"2022-12-07T15:41:28.114635Z","shell.execute_reply.started":"2022-12-07T15:41:28.108539Z","shell.execute_reply":"2022-12-07T15:41:28.113268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"See how many images are available for individuals with cancer and no cancer","metadata":{}},{"cell_type":"code","source":"num_img_cancer=[]\nnum_img_no_cancer=[]\n\nfor i in range(0,len(patient_id_cancer)):\n    num_img_cancer.append(number_of_img_for_patient(patient_id_cancer[i]))\n\nfor i in range(0,len(patient_id_no_cancer)):\n    num_img_no_cancer.append(number_of_img_for_patient(patient_id_no_cancer[i]))","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:49:07.749395Z","iopub.execute_input":"2022-12-07T15:49:07.749829Z","iopub.status.idle":"2022-12-07T15:49:14.071391Z","shell.execute_reply.started":"2022-12-07T15:49:07.749795Z","shell.execute_reply":"2022-12-07T15:49:14.070022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(num_img_cancer,alpha=0.5,label='cancer')\nplt.hist(num_img_no_cancer,alpha=0.5,label='No cancer')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:41:59.740476Z","iopub.execute_input":"2022-12-07T15:41:59.741478Z","iopub.status.idle":"2022-12-07T15:42:00.083257Z","shell.execute_reply.started":"2022-12-07T15:41:59.741433Z","shell.execute_reply":"2022-12-07T15:42:00.08187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some ids with cancer","metadata":{}},{"cell_type":"code","source":"show_images_for_patient(9707)","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:56:22.358983Z","iopub.execute_input":"2022-12-07T15:56:22.359427Z","iopub.status.idle":"2022-12-07T15:56:27.9644Z","shell.execute_reply.started":"2022-12-07T15:56:22.35939Z","shell.execute_reply":"2022-12-07T15:56:27.963078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get Ids of images where there are 4 images per patient with cancer (for subsequent comparisons)","metadata":{}},{"cell_type":"code","source":"show_images_for_patient(9989)","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:56:57.729555Z","iopub.execute_input":"2022-12-07T15:56:57.729976Z","iopub.status.idle":"2022-12-07T15:56:58.403186Z","shell.execute_reply.started":"2022-12-07T15:56:57.72994Z","shell.execute_reply":"2022-12-07T15:56:58.401277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#four_imgs_cancer_id[0]\ntrain_data_cancer.iloc[four_imgs_cancer_id[0]]\n#train_data_cancer.iloc[four_imgs_cancer_id[0]].patient_id","metadata":{"execution":{"iopub.status.busy":"2022-12-07T15:48:13.325143Z","iopub.execute_input":"2022-12-07T15:48:13.325593Z","iopub.status.idle":"2022-12-07T15:48:13.335941Z","shell.execute_reply.started":"2022-12-07T15:48:13.325549Z","shell.execute_reply":"2022-12-07T15:48:13.334697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"....to be continued!","metadata":{}}]}