{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"},{"sourceId":4619402,"sourceType":"datasetVersion","datasetId":2688773},{"sourceId":10866548,"sourceType":"datasetVersion","datasetId":6750741},{"sourceId":269647,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":230751,"modelId":252513}],"dockerImageVersionId":30301,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -qU python-gdcm pydicom pylibjpeg\n\n# Data manipulation and visualization libraries\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport cv2\n\n# File and directory handling libraries\nimport pydicom\nfrom os import listdir\n\n# Statistical and data processing libraries\nfrom scipy.stats import mode, skew\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.mixture import GaussianMixture\n\n# Progress tracking and warning suppression libraries\nfrom tqdm.notebook import trange\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\n\n# Set seaborn style\nsns.set_style('darkgrid')","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-02-27T04:52:45.40756Z","iopub.execute_input":"2025-02-27T04:52:45.408496Z","iopub.status.idle":"2025-02-27T04:56:42.392555Z","shell.execute_reply.started":"2025-02-27T04:52:45.408379Z","shell.execute_reply":"2025-02-27T04:56:42.391474Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"filepath = '/kaggle/input/rsna-breast-cancer-detection/train.csv'\ndata = pd.read_csv(filepath)\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:42.395071Z","iopub.execute_input":"2025-02-27T04:56:42.395409Z","iopub.status.idle":"2025-02-27T04:56:42.527411Z","shell.execute_reply.started":"2025-02-27T04:56:42.395379Z","shell.execute_reply":"2025-02-27T04:56:42.526382Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:42.528552Z","iopub.execute_input":"2025-02-27T04:56:42.52887Z","iopub.status.idle":"2025-02-27T04:56:42.549289Z","shell.execute_reply.started":"2025-02-27T04:56:42.528837Z","shell.execute_reply":"2025-02-27T04:56:42.548213Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_patients = data['patient_id'].nunique()\nmin_patient_age = int(data['age'].min())\nmax_patient_age = int(data['age'].max())\ngroupby_id = data.groupby('patient_id')['cancer'].max()\nn_negative = (groupby_id == 0).sum()\nn_positive = (groupby_id == 1).sum()\n\nprint(f\"There are {num_patients} different patients in the train set.\\n\")\nprint(f\"The younger patient is {min_patient_age} years old.\")\nprint(f\"The older patient is {max_patient_age} years old.\\n\")\nprint(f\"There are {n_negative} patients negative to breast cancer. Ratio = {n_negative / num_patients}\")\nprint(f\"There are {n_positive} patients positive to breast cancer. Ratio = {n_positive / num_patients}\")","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:42.550831Z","iopub.execute_input":"2025-02-27T04:56:42.551936Z","iopub.status.idle":"2025-02-27T04:56:42.576461Z","shell.execute_reply.started":"2025-02-27T04:56:42.551895Z","shell.execute_reply":"2025-02-27T04:56:42.57538Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ages = data.groupby('patient_id')['age'].apply(lambda x: x.unique()[0])\ncancer_ages = data[data['cancer'] == 1].groupby('patient_id')['age'].apply(lambda x: x.unique()[0])\nno_cancer_ages = data[data['cancer'] == 0].groupby('patient_id')['age'].apply(lambda x: x.unique()[0])\n\nplt.figure(figsize=(16, 10))\n\nplt.subplot(1, 2, 1)\nsns.histplot(ages, bins=63, color='orange', kde=True)\nplt.title(\"All the patient\")\nplt.xlim(33, 89)\n\nplt.subplot(2, 2, 2)\nsns.histplot(cancer_ages, bins=51, color='red', kde=True)\nplt.title(\"Patients with cancer\")\nplt.xlim(33, 89)\n\nplt.subplot(2, 2, 4)\nsns.histplot(no_cancer_ages, bins=63, color='green', kde=True)\nplt.title(\"Patients without cancer\")\nplt.xlim(33, 89)\n\nplt.suptitle(\"Age distribution of the patients\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:42.577656Z","iopub.execute_input":"2025-02-27T04:56:42.577999Z","iopub.status.idle":"2025-02-27T04:56:44.882262Z","shell.execute_reply.started":"2025-02-27T04:56:42.577968Z","shell.execute_reply":"2025-02-27T04:56:44.881099Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Statistics\nprint(\"Mean:\", ages.mean())\nprint(\"Std:\", ages.std())\nprint(\"Q1:\", ages.quantile(0.25))\nprint(\"Median:\", ages.median())\nprint(\"Q3:\", ages.quantile(0.75))\nprint(\"Mode:\", ages.mode()[0])","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:44.883903Z","iopub.execute_input":"2025-02-27T04:56:44.884367Z","iopub.status.idle":"2025-02-27T04:56:44.899583Z","shell.execute_reply.started":"2025-02-27T04:56:44.884325Z","shell.execute_reply":"2025-02-27T04:56:44.898518Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_images_per_patient = data['patient_id'].value_counts()\nplt.figure(figsize=(16, 6))\nsns.countplot(n_images_per_patient, palette='Reds_r')\nplt.title(\"Number of images taken per patients\")\nplt.xlabel('Number of images taken')\nplt.ylabel('Count of patients')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:44.903908Z","iopub.execute_input":"2025-02-27T04:56:44.904323Z","iopub.status.idle":"2025-02-27T04:56:45.114757Z","shell.execute_reply.started":"2025-02-27T04:56:44.904292Z","shell.execute_reply":"2025-02-27T04:56:45.11359Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 3, figsize=(16, 8))\nsns.countplot(data['laterality'], palette='Blues_r', ax=ax[0, 0])\nsns.countplot(data['implant'], palette='Greens_r', ax=ax[0, 1])\nsns.countplot(data['difficult_negative_case'], palette='Reds_r', ax=ax[0, 2])\nsns.countplot(data['view'], palette='Oranges_r', ax=ax[1, 0])\nsns.countplot(data['density'], palette='Purples_r', order=['A', 'B', 'C', 'D'], ax=ax[1, 1])\nsns.countplot(data['site_id'], palette='Greys_r', ax=ax[1, 2])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:45.115938Z","iopub.execute_input":"2025-02-27T04:56:45.116242Z","iopub.status.idle":"2025-02-27T04:56:45.873807Z","shell.execute_reply.started":"2025-02-27T04:56:45.116213Z","shell.execute_reply":"2025-02-27T04:56:45.872625Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"biopsy_counts = data.groupby('cancer')['biopsy'].value_counts().unstack().fillna(0)\nbiopsy_perc = biopsy_counts.transpose() / biopsy_counts.sum(axis=1)\n\nfig, ax = plt.subplots(1, 2, figsize=(10, 4))\nsns.countplot(data['cancer'], palette='Greens', ax=ax[0])\nsns.heatmap(biopsy_perc, square=True, annot=True, fmt='.1%', cmap='Blues', ax=ax[1])\nax[0].set_title(\"Number of images showing cancer\")\nax[1].set_title(\"Percentage of images\\nresulting in a biopsy\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:45.875371Z","iopub.execute_input":"2025-02-27T04:56:45.876208Z","iopub.status.idle":"2025-02-27T04:56:46.220166Z","shell.execute_reply.started":"2025-02-27T04:56:45.876169Z","shell.execute_reply":"2025-02-27T04:56:46.219071Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(16, 4))\nsns.countplot(data[data['cancer'] == True]['invasive'], ax=ax[0], palette='Reds')\nsns.countplot(data[data['cancer'] == False]['BIRADS'], order=[0, 1, 2], ax=ax[1], palette='Blues')\nsns.countplot(data[data['cancer'] == True]['BIRADS'], order=[0, 1, 2], ax=ax[2], palette='Blues')\nax[0].set_title(\"Count of invasive cancer images\")\nax[1].set_title(\"BIRADS for healthy images\")\nax[2].set_title(\"BIRADS for cancer images\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:46.221302Z","iopub.execute_input":"2025-02-27T04:56:46.221582Z","iopub.status.idle":"2025-02-27T04:56:46.621495Z","shell.execute_reply.started":"2025-02-27T04:56:46.221556Z","shell.execute_reply":"2025-02-27T04:56:46.620364Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(14, 5))\nsns.countplot(data['machine_id'])\nplt.title(\"Count of images taken by machine ID\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:46.623151Z","iopub.execute_input":"2025-02-27T04:56:46.623599Z","iopub.status.idle":"2025-02-27T04:56:46.857642Z","shell.execute_reply.started":"2025-02-27T04:56:46.623555Z","shell.execute_reply":"2025-02-27T04:56:46.85642Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_path = '/kaggle/input/rsna-breast-cancer-detection/train_images'\ntest_path = '/kaggle/input/rsna-breast-cancer-detection/test_images'","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:46.85892Z","iopub.execute_input":"2025-02-27T04:56:46.859274Z","iopub.status.idle":"2025-02-27T04:56:46.86415Z","shell.execute_reply.started":"2025-02-27T04:56:46.859245Z","shell.execute_reply":"2025-02-27T04:56:46.863006Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_patient_scans(path, patient_id):\n    patient_path = path + '/' + str(patient_id)\n    return [pydicom.dcmread(patient_path + '/' + file) for file in listdir(patient_path)]","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:46.865866Z","iopub.execute_input":"2025-02-27T04:56:46.866766Z","iopub.status.idle":"2025-02-27T04:56:46.87538Z","shell.execute_reply.started":"2025-02-27T04:56:46.86672Z","shell.execute_reply":"2025-02-27T04:56:46.874334Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load all scans of the twelfth patient\npatient_ids = data['patient_id'].unique()\nscans = load_patient_scans(train_path, patient_ids[11])","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:46.87664Z","iopub.execute_input":"2025-02-27T04:56:46.877022Z","iopub.status.idle":"2025-02-27T04:56:47.887032Z","shell.execute_reply.started":"2025-02-27T04:56:46.876991Z","shell.execute_reply":"2025-02-27T04:56:47.885884Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Look at raw pixelarrays\nfig, ax = plt.subplots(1, 2, figsize=(20, 5))\nim = ax[0].imshow(scans[0].pixel_array, cmap='jet')\nax[0].grid(False)\nfig.colorbar(im, ax=ax[0])\nsns.histplot(scans[0].pixel_array.flatten(), ax=ax[1], bins=50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:47.888408Z","iopub.execute_input":"2025-02-27T04:56:47.88875Z","iopub.status.idle":"2025-02-27T04:56:48.425017Z","shell.execute_reply.started":"2025-02-27T04:56:47.888716Z","shell.execute_reply":"2025-02-27T04:56:48.422344Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_scan_info():\n    modes, rows, cols = [], [], []\n    machine_ids = data['machine_id'].unique()\n    for m_id in machine_ids:\n        m_id_modes, m_id_rows, m_id_cols = [], [], []\n        print(f\"Machine id {m_id} in progress\")\n        patient_ids = data[data['machine_id'] == m_id]['patient_id'].unique()\n        for n in range(50):\n            try:\n                scan = load_patient_scans(train_path, patient_ids[n])[0]\n                m_id_modes.append(mode(scan.pixel_array.flatten())[0][0])\n                m_id_rows.append(scan.Rows)\n                m_id_cols.append(scan.Columns)\n            except IndexError:\n                break\n        modes.append(m_id_modes)\n        rows.append(m_id_rows)\n        cols.append(m_id_cols)\n    return modes, rows, cols","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:58:42.064646Z","iopub.execute_input":"2025-02-27T04:58:42.065542Z","iopub.status.idle":"2025-02-27T04:58:42.074099Z","shell.execute_reply.started":"2025-02-27T04:58:42.065504Z","shell.execute_reply":"2025-02-27T04:58:42.072683Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"modes, rows, cols = get_scan_info()\n\nmachine_ids = data['machine_id'].unique()\nmedians = [np.median(x) for x in modes]\nstds = [np.std(x) for x in modes]\nrows = [np.mean(x) for x in rows]\ncols = [np.mean(x) for x in cols]\ndf = pd.DataFrame(data={'Machine ID': machine_ids, 'Mode (median)': medians, 'Mode (std)': stds, 'Rows (mean)': rows, 'Cols (mean)': cols})\ndf.astype(int).set_index('Machine ID').T","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:58:46.799758Z","iopub.execute_input":"2025-02-27T04:58:46.80018Z","iopub.status.idle":"2025-02-27T05:02:23.093626Z","shell.execute_reply.started":"2025-02-27T04:58:46.800134Z","shell.execute_reply":"2025-02-27T05:02:23.092159Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(22, 8))\nfor i, m_id in enumerate(machine_ids):\n    patient_ids = data[data['machine_id'] == m_id]['patient_id'].unique()\n    scan = load_patient_scans(train_path, patient_ids[0])[0] # Load first scan of first patient\n    plt.subplot(2, 5, i+1)\n    plt.imshow(scan.pixel_array, cmap='jet')\n    plt.title(f\"Machine {m_id}\")\n    plt.colorbar()\n    plt.grid(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.429867Z","iopub.status.idle":"2025-02-27T04:56:48.430246Z","shell.execute_reply.started":"2025-02-27T04:56:48.430064Z","shell.execute_reply":"2025-02-27T04:56:48.430081Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"m_id_implants = data[data['implant'] == 1]['machine_id'].unique()\nprint(\"Scans showing implents are from machines\", m_id_implants)","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.431682Z","iopub.status.idle":"2025-02-27T04:56:48.432125Z","shell.execute_reply.started":"2025-02-27T04:56:48.431932Z","shell.execute_reply":"2025-02-27T04:56:48.431952Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"patient_ids = data[data['implant'] == 1]['patient_id'].unique()\n\n# Display scans showing implants\nplt.figure(figsize=(22, 8))\nfor i in range(10):\n    scan = load_patient_scans(train_path, patient_ids[i])[0] # Load first scan of the patient\n    plt.subplot(2, 5, i+1)\n    plt.imshow(scan.pixel_array, cmap='jet')\n    plt.title(f\"Patient {patient_ids[i]}\")\n    plt.grid(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.434074Z","iopub.status.idle":"2025-02-27T04:56:48.434445Z","shell.execute_reply.started":"2025-02-27T04:56:48.434271Z","shell.execute_reply":"2025-02-27T04:56:48.434289Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display scans of patient 13095\nplt.figure(figsize=(22, 8))\nscans = load_patient_scans(train_path, 13095)\nfor i in range(10):\n    plt.subplot(2, 5, i+1)\n    plt.imshow(scans[i].pixel_array, cmap='jet')\n    plt.grid(False)\nplt.suptitle(\"All scans of patient 13095\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.435557Z","iopub.status.idle":"2025-02-27T04:56:48.435938Z","shell.execute_reply.started":"2025-02-27T04:56:48.43573Z","shell.execute_reply":"2025-02-27T04:56:48.435745Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Choose to display images with or without cancer\ndef display_cancer_or_not(cancer=True):\n    cancer_scans = data[data['cancer'] == int(cancer)].sample(frac=1, random_state=0)\n    plt.figure(figsize=(22, 10))\n    for i in trange(10):\n        patient = str(cancer_scans.iloc[i][['patient_id']][0])\n        file = str(cancer_scans.iloc[i][['image_id']][0]) + '.dcm'\n        scan = pydicom.dcmread(train_path + '/' + patient + '/' + file)\n        plt.subplot(2, 5, i+1)\n        plt.imshow(scan.pixel_array, cmap='jet')\n        plt.title(f\"Patient {patient}\\nScan {file}\")\n        plt.grid(False)\n    plt.suptitle(f\"Cancer = {cancer}\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.43753Z","iopub.status.idle":"2025-02-27T04:56:48.437905Z","shell.execute_reply.started":"2025-02-27T04:56:48.437703Z","shell.execute_reply":"2025-02-27T04:56:48.43772Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Images with cancer\ndisplay_cancer_or_not(cancer=True)","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.44021Z","iopub.status.idle":"2025-02-27T04:56:48.440643Z","shell.execute_reply.started":"2025-02-27T04:56:48.440412Z","shell.execute_reply":"2025-02-27T04:56:48.440436Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Images without cancer\ndisplay_cancer_or_not(cancer=False)","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.441584Z","iopub.status.idle":"2025-02-27T04:56:48.441983Z","shell.execute_reply.started":"2025-02-27T04:56:48.441771Z","shell.execute_reply":"2025-02-27T04:56:48.441788Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"source = '../input/rsna-breast-cancer-256-pngs/'\ndata['path'] = source + data['patient_id'].astype(str) + \"_\" + data['image_id'].astype(str) + \".png\"","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.443988Z","iopub.status.idle":"2025-02-27T04:56:48.444337Z","shell.execute_reply.started":"2025-02-27T04:56:48.444174Z","shell.execute_reply":"2025-02-27T04:56:48.44419Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# All patients of machine ID 21\npatients = data[data.machine_id == 21].patient_id.unique()\n\nmeans, medians, stds, skews, paths, labels = [], [], [], [], [], []\n\nfor i in trange(len(patients)):\n    path = data[data['patient_id'] == patients[i]]['path'].iloc[0]\n    label = data[data['patient_id'] == patients[i]]['cancer'].iloc[0]\n    img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)\n    means.append(np.mean(img))\n    medians.append(np.median(img))\n    stds.append(np.std(img))\n    skews.append(skew(img, axis=None))\n    paths.append(path)\n    labels.append(label)\n    \nstats = pd.DataFrame()\nstats['mean'] = means\nstats['median'] = medians\nstats['std'] = stds\nstats['skew'] = skews\nstats['path'] = paths\nstats['cancer'] = labels\nstats.head()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.446157Z","iopub.status.idle":"2025-02-27T04:56:48.446517Z","shell.execute_reply.started":"2025-02-27T04:56:48.446346Z","shell.execute_reply":"2025-02-27T04:56:48.446363Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig = px.scatter_3d(stats, x='mean', y='std', z='skew', color='median')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.447724Z","iopub.status.idle":"2025-02-27T04:56:48.448128Z","shell.execute_reply.started":"2025-02-27T04:56:48.447943Z","shell.execute_reply":"2025-02-27T04:56:48.447961Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.heatmap(np.abs(stats.corr()), annot=True,\n            square=True, vmin=0, vmax=1, cmap='Blues')\nplt.title(\"Correlation matrix of the statistics\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.449383Z","iopub.status.idle":"2025-02-27T04:56:48.44971Z","shell.execute_reply.started":"2025-02-27T04:56:48.449548Z","shell.execute_reply":"2025-02-27T04:56:48.449564Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = StandardScaler()\nX = stats.drop(['path', 'cancer'], axis=1)\nX = scaler.fit_transform(X)\n\ngmm = GaussianMixture(n_components=4, random_state=0)\nstats['cluster_label'] = gmm.fit_predict(X)","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.450673Z","iopub.status.idle":"2025-02-27T04:56:48.451046Z","shell.execute_reply.started":"2025-02-27T04:56:48.45087Z","shell.execute_reply":"2025-02-27T04:56:48.450888Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig = px.scatter_3d(stats, x='mean', y='std', z='skew', color='cluster_label')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.452755Z","iopub.status.idle":"2025-02-27T04:56:48.453174Z","shell.execute_reply.started":"2025-02-27T04:56:48.452995Z","shell.execute_reply":"2025-02-27T04:56:48.453014Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(10,4,figsize=(20,40))\nfor l in range(4):\n    paths = stats[stats['cluster_label'] == l]['path'].values[0:10]\n    cancers = stats[stats['cluster_label'] == l]['cancer'].values[0:10]\n    for n in range(10):\n        img = cv2.imread(paths[n], cv2.IMREAD_GRAYSCALE)\n        ax[n,l].imshow(img, cmap='jet')\n        ax[n,l].set_title(f\"label {l} / cancer {cancers[n]}\")\n        ax[n,l].axis('off')","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.454523Z","iopub.status.idle":"2025-02-27T04:56:48.454896Z","shell.execute_reply.started":"2025-02-27T04:56:48.454696Z","shell.execute_reply":"2025-02-27T04:56:48.454712Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats['logL'] = gmm.score_samples(X)\nstats['logL'].quantile(0.005)","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.456332Z","iopub.status.idle":"2025-02-27T04:56:48.456907Z","shell.execute_reply.started":"2025-02-27T04:56:48.45658Z","shell.execute_reply":"2025-02-27T04:56:48.456606Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outliers = stats[stats['logL'] < -10].sort_values(by='logL')\nprint(f\"{len(outliers)} outliers have been found!\")","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.459038Z","iopub.status.idle":"2025-02-27T04:56:48.459553Z","shell.execute_reply.started":"2025-02-27T04:56:48.459286Z","shell.execute_reply":"2025-02-27T04:56:48.459312Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outliers['path'].iloc[0]","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.462021Z","iopub.status.idle":"2025-02-27T04:56:48.462542Z","shell.execute_reply.started":"2025-02-27T04:56:48.462273Z","shell.execute_reply":"2025-02-27T04:56:48.462299Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(20, 8))\nfor i in range(10):\n    plt.subplot(2, 5, i+1)\n    img = cv2.imread(outliers['path'].iloc[i], cv2.IMREAD_GRAYSCALE)\n    plt.imshow(img, cmap='jet')\n    plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2025-02-27T04:56:48.464821Z","iopub.status.idle":"2025-02-27T04:56:48.465268Z","shell.execute_reply.started":"2025-02-27T04:56:48.465084Z","shell.execute_reply":"2025-02-27T04:56:48.465105Z"},"trusted":true},"outputs":[],"execution_count":null}]}