{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \n\ntry:\n    import pylibjpeg\nexcept:\n   !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:09.893442Z","iopub.execute_input":"2023-04-18T14:35:09.894077Z","iopub.status.idle":"2023-04-18T14:35:39.450944Z","shell.execute_reply.started":"2023-04-18T14:35:09.894035Z","shell.execute_reply":"2023-04-18T14:35:39.449659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport numpy as np # linear algebra\nimport cv2 as cv\n\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras import layers, models, optimizers\n\nfrom keras import backend as K\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, Callback, ReduceLROnPlateau\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nimport keras\nimport os\nimport pydicom as dicom\nimport imageio","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:39.453563Z","iopub.execute_input":"2023-04-18T14:35:39.454302Z","iopub.status.idle":"2023-04-18T14:35:39.462783Z","shell.execute_reply.started":"2023-04-18T14:35:39.454257Z","shell.execute_reply":"2023-04-18T14:35:39.46139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(r'/kaggle/input/rsna-breast-cancer-detection/train.csv')\n# clean data\ndf.drop(['site_id','machine_id'], axis=1, inplace=True)\ndf.dropna(subset=['age'], inplace=True)\ndf = df.assign(age=df[\"age\"]/100)\ndf['view'].replace({'CC': 0,'MLO': 1}, inplace=True) ## replace 0 with mean value\ndf['laterality'].replace({'L': 0,'R': 1}, inplace=True) ## replace 0 with mean value\n# df['density'].replace({'A': 1,'B': 2,'C':3,'D':4}, inplace=True) ## replace 0 with mean value\ndf[\"difficult_negative_case\"] = df[\"difficult_negative_case\"].astype(int)\ndf.loc[(df['cancer'] == 1) & (df['biopsy'] == 1) & (df['BIRADS'].isnull()),'BIRADS'] = 0\ndf.dropna(subset=['BIRADS'], inplace=True)\n# df.loc[(df['cancer'] == 1) & (df['biopsy'] == 0) & (df['BIRADS'].isnull()),'BIRADS'] = 0\n\n\nx1 = df.loc[(df['cancer'] == 0),'age']\ny1 = df.loc[(df['cancer'] == 0),'cancer']\n\nx2 = df.loc[(df['cancer'] == 1),'age']\ny2 = df.loc[(df['cancer'] == 1),'cancer']\n\ndf['cancer'] = df['cancer'].astype(str)\ndf['implant'].value_counts(dropna=False)\nplt.hist(df['cancer'])","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:39.464553Z","iopub.execute_input":"2023-04-18T14:35:39.464993Z","iopub.status.idle":"2023-04-18T14:35:39.790481Z","shell.execute_reply.started":"2023-04-18T14:35:39.464956Z","shell.execute_reply":"2023-04-18T14:35:39.789522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_CC_Cancer = df[(df.cancer == \"0\")&(df.view == 0)].sample(1000)\ndf_MLO_Cancer = df[(df.cancer == \"0\")&(df.view == 1)].sample(1000)\ndf_is_Cancer = df[df.cancer == \"1\"]","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:39.794441Z","iopub.execute_input":"2023-04-18T14:35:39.79473Z","iopub.status.idle":"2023-04-18T14:35:39.823346Z","shell.execute_reply.started":"2023-04-18T14:35:39.794702Z","shell.execute_reply":"2023-04-18T14:35:39.822421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample_cancer = pd.concat([df_CC_Cancer, df_MLO_Cancer,df_is_Cancer]).sample(frac=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:39.824716Z","iopub.execute_input":"2023-04-18T14:35:39.825082Z","iopub.status.idle":"2023-04-18T14:35:39.837903Z","shell.execute_reply.started":"2023-04-18T14:35:39.825045Z","shell.execute_reply":"2023-04-18T14:35:39.83682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2 = pd.read_csv(r'/kaggle/input/rsna-breast-cancer-detection/test.csv')\n# clean data\ndf2.drop(['prediction_id','machine_id','site_id'], axis=1, inplace=True)\ndf2 = df2.assign(age=df2[\"age\"]/100)\ndf2['view'].replace({'CC': 0,'MLO': 1}, inplace=True) ## replace 0 with mean value\n# df2['laterality'].replace({'L': 0,'R': 1}, inplace=True) ## replace 0 with mean value","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:39.839558Z","iopub.execute_input":"2023-04-18T14:35:39.840031Z","iopub.status.idle":"2023-04-18T14:35:39.852543Z","shell.execute_reply.started":"2023-04-18T14:35:39.839972Z","shell.execute_reply":"2023-04-18T14:35:39.851509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2[\"patient_id\"] = df2[\"patient_id\"].astype(str)\ndf2[\"image_id\"] = df2[\"image_id\"].astype(str)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:39.854037Z","iopub.execute_input":"2023-04-18T14:35:39.854389Z","iopub.status.idle":"2023-04-18T14:35:39.861768Z","shell.execute_reply.started":"2023-04-18T14:35:39.854352Z","shell.execute_reply":"2023-04-18T14:35:39.860604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_PATH = '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs/train_images_processed'\nINPUT_PATH_TEST = '/kaggle/input/rsna-breast-cancer-detection/test_images'\ndf_sample_cancer['Image'] = df.apply(lambda x: f'{INPUT_PATH}/{x[\"patient_id\"]}/{x[\"image_id\"]}.png', axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:39.863619Z","iopub.execute_input":"2023-04-18T14:35:39.864546Z","iopub.status.idle":"2023-04-18T14:35:40.217573Z","shell.execute_reply.started":"2023-04-18T14:35:39.864511Z","shell.execute_reply":"2023-04-18T14:35:40.216558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_files = np.array(df_sample_cancer['Image'].tolist())\ny_train = np.array(df_sample_cancer['cancer'].tolist())","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:40.219066Z","iopub.execute_input":"2023-04-18T14:35:40.219529Z","iopub.status.idle":"2023-04-18T14:35:40.228377Z","shell.execute_reply.started":"2023-04-18T14:35:40.219487Z","shell.execute_reply":"2023-04-18T14:35:40.227297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_classes = np.array(df_sample_cancer['cancer'].unique())\nn_classes = n_classes.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:40.233468Z","iopub.execute_input":"2023-04-18T14:35:40.233747Z","iopub.status.idle":"2023-04-18T14:35:40.239897Z","shell.execute_reply.started":"2023-04-18T14:35:40.233721Z","shell.execute_reply":"2023-04-18T14:35:40.23889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_classes","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:40.241157Z","iopub.execute_input":"2023-04-18T14:35:40.242141Z","iopub.status.idle":"2023-04-18T14:35:40.252745Z","shell.execute_reply.started":"2023-04-18T14:35:40.242103Z","shell.execute_reply":"2023-04-18T14:35:40.251738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recall_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef precision_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef f1_m(y_true, y_pred):\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:40.254146Z","iopub.execute_input":"2023-04-18T14:35:40.254683Z","iopub.status.idle":"2023-04-18T14:35:40.263994Z","shell.execute_reply.started":"2023-04-18T14:35:40.254638Z","shell.execute_reply":"2023-04-18T14:35:40.262888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_fold = 0\n\ny_test = []\n\nfolds = KFold(n_splits=2, shuffle=True, random_state=1).split(X_train_files, y_train)\nfor train_index, test_index in folds:\n    X_train_files_fold = X_train_files[train_index]\n    y_train_fold = y_train[train_index]\n    X_val_files_fold = X_train_files[test_index]\n    y_val_fold = np.array(y_train[test_index])\n        \n    train_df = pd.DataFrame(list(zip(X_train_files_fold, y_train_fold)), columns = ['foldername', 'cancer'])\n    val_df = pd.DataFrame(list(zip(X_val_files_fold, y_val_fold)), columns = ['foldername', 'cancer'])\n    \n#     print(len(train_df[train_df.cancer == '0']),len(train_df[train_df.cancer == '1']))\n#     print(len(val_df[val_df.cancer == '0']),len(val_df[val_df.cancer == '1']))\n\n              \n    train_datagen = ImageDataGenerator(\n        rescale=1./255,\n    )\n    \n    train_generator = train_datagen.flow_from_dataframe(\n        dataframe = train_df,\n        train_dir=None,\n        x_col ='foldername',\n        y_col ='cancer',\n        target_size=(256, 256),\n        class_mode='binary',\n        batch_size=128,\n        classes=n_classes,\n    )\n    \n    val_datagen = ImageDataGenerator(\n        rescale=1./255\n    )\n    \n    val_generator = val_datagen.flow_from_dataframe(\n        dataframe = val_df,\n        train_dir=None,\n        x_col='foldername',\n        y_col='cancer',\n        target_size=(256, 256),\n        class_mode='binary',\n        batch_size=128,\n        classes=n_classes,\n    )\n#     test_datagen = ImageDataGenerator (rescale = 1./255)\n    \n#     test_generator = test_datagen.flow_from_directory(\n#         directory=INPUT_PATH_TEST,\n#         target_size=(256, 256),\n#         batch_size=1,\n#         class_mode=None,\n#         shuffle=False\n#     )\n    model_path_of_fold = os.path.join('', 'weights_of_fold_' + str(num_fold) + '.h5')\n\n#   create model\n    model = models.Sequential()\n    model.add(layers.Conv2D(32, (3, 3), input_shape=(256, 256, 3)))\n#     model.add(layers.BatchNormalization())\n    model.add(layers.Activation(\"relu\"))\n\n    model.add(layers.MaxPooling2D((2, 2)))\n\n    model.add(layers.Conv2D(64, (3, 3)))\n#     model.add(layers.BatchNormalization())\n    model.add(layers.Activation(\"relu\"))\n\n    model.add(layers.MaxPooling2D((2, 2)))\n\n#     model.add(layers.Conv2D(128, (3, 3)))\n#     model.add(layers.BatchNormalization())\n#     model.add(layers.Activation(\"relu\"))\n\n#     model.add(layers.MaxPooling2D((2, 2)))\n\n    model.add(layers.Flatten())\n#     model.add(layers.Dropout(0.4))\n    model.add(layers.Dense(32))\n#     model.add(layers.BatchNormalization())\n    model.add(layers.Activation(\"relu\"))\n\n    model.add(layers.Dense(1))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation(\"sigmoid\"))\n\n    model.compile(\n        loss='binary_crossentropy',\n        optimizer=optimizers.Adam(learning_rate=0.0035, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0),\n        metrics=[f1_m])\n    callbacks = [\n        ModelCheckpoint(model_path_of_fold, monitor='val_f1_m', save_best_only=True, mode='max'),\n        ReduceLROnPlateau(monitor='loss', factor=0.1, patience=3, mode='min', min_lr=1e-5)\n    ]\n    \n    history = model.fit(train_generator, epochs=5, validation_data=val_generator, callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:35:40.265542Z","iopub.execute_input":"2023-04-18T14:35:40.265927Z","iopub.status.idle":"2023-04-18T14:38:00.261815Z","shell.execute_reply.started":"2023-04-18T14:35:40.265884Z","shell.execute_reply":"2023-04-18T14:38:00.260666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2['Image'] = df2.apply(lambda x: f'{INPUT_PATH_TEST}/{x[\"patient_id\"]}/{x[\"image_id\"]}.dcm', axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:38:00.263334Z","iopub.execute_input":"2023-04-18T14:38:00.26405Z","iopub.status.idle":"2023-04-18T14:38:00.271475Z","shell.execute_reply.started":"2023-04-18T14:38:00.26399Z","shell.execute_reply":"2023-04-18T14:38:00.270284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_generator(df, image_size):\n    for _, row in df.iterrows():\n        dcm_data = dicom.read_file(row['Image'])\n        pixel_data = dcm_data.pixel_array\n        pixel_data = (pixel_data - pixel_data.min()) / (pixel_data.max() - pixel_data.min()) * 255\n        pixel_data = pixel_data.astype('uint8')\n        resized_image = cv.resize(np.array(pixel_data), image_size, interpolation=cv.INTER_LINEAR)\n        resized_image = cv.cvtColor(resized_image, cv.COLOR_GRAY2RGB)\n        yield resized_image\n\nimage_size = (256, 256)\nimage_list = np.array(list(image_generator(df2, image_size)))","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:48:37.534347Z","iopub.execute_input":"2023-04-18T14:48:37.535358Z","iopub.status.idle":"2023-04-18T14:48:39.654261Z","shell.execute_reply.started":"2023-04-18T14:48:37.535302Z","shell.execute_reply":"2023-04-18T14:48:39.653183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_images = np.array(image_list)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:38:02.487384Z","iopub.execute_input":"2023-04-18T14:38:02.487742Z","iopub.status.idle":"2023-04-18T14:38:02.496026Z","shell.execute_reply.started":"2023-04-18T14:38:02.487704Z","shell.execute_reply":"2023-04-18T14:38:02.495062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# resized_images = []\n# for image in test_images:\n#     resized_image = cv.resize(image, (256, 256), interpolation=cv.INTER_LINEAR)\n#     resized_image = cv.cvtColor(resized_image, cv.COLOR_GRAY2RGB) # Convert to RGB\n#     resized_images.append(resized_image)\n\n# # Convert list of resized images to array\n# resized_test_images = np.array(resized_images)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:38:02.497722Z","iopub.execute_input":"2023-04-18T14:38:02.498139Z","iopub.status.idle":"2023-04-18T14:38:02.505029Z","shell.execute_reply.started":"2023-04-18T14:38:02.498102Z","shell.execute_reply":"2023-04-18T14:38:02.504056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_datagen = ImageDataGenerator(rescale=1./255)\ntest_generator = test_datagen.flow(image_list, batch_size=1, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:48:43.976594Z","iopub.execute_input":"2023-04-18T14:48:43.97722Z","iopub.status.idle":"2023-04-18T14:48:43.983309Z","shell.execute_reply.started":"2023-04-18T14:48:43.977181Z","shell.execute_reply":"2023-04-18T14:48:43.982074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_test = model.predict(test_generator)\ny_test.append(p_test)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:38:02.518323Z","iopub.execute_input":"2023-04-18T14:38:02.519143Z","iopub.status.idle":"2023-04-18T14:38:02.671244Z","shell.execute_reply.started":"2023-04-18T14:38:02.519105Z","shell.execute_reply":"2023-04-18T14:38:02.670256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = np.array(p_test)\nresults = pd.DataFrame()\nresults['prediction_id'] = df2.apply(lambda x: f'{x[\"patient_id\"]}_{x[\"laterality\"]}', axis=1)\nresults['cancer'] = result\nresults = results.groupby(['prediction_id'],as_index=False)['cancer'].max()","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:38:02.672659Z","iopub.execute_input":"2023-04-18T14:38:02.673123Z","iopub.status.idle":"2023-04-18T14:38:02.686121Z","shell.execute_reply.started":"2023-04-18T14:38:02.673084Z","shell.execute_reply":"2023-04-18T14:38:02.685034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-18T14:38:02.687691Z","iopub.execute_input":"2023-04-18T14:38:02.688163Z","iopub.status.idle":"2023-04-18T14:38:02.697294Z","shell.execute_reply.started":"2023-04-18T14:38:02.688124Z","shell.execute_reply":"2023-04-18T14:38:02.696485Z"},"trusted":true},"execution_count":null,"outputs":[]}]}