{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Notebook from a kaggle novice❗\n\nIf you like this and think it helpful, please give a like 👍. Many thanks ！🏄‍\n","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:11:03.457277Z"}}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport matplotlib.pyplot as plt \nimport cv2 as cv\nfrom path import Path\nimport os \nimport glob\nimport tensorflow_hub as hub\nimport os \nimport pydicom as dicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras import layers\nfrom tqdm import tqdm\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nfrom tensorflow.keras.utils import to_categorical\nfrom pydicom import dcmread\nimport nibabel as nib","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:18:22.249697Z","iopub.execute_input":"2022-07-31T03:18:22.250638Z","iopub.status.idle":"2022-07-31T03:18:30.701796Z","shell.execute_reply.started":"2022-07-31T03:18:22.250541Z","shell.execute_reply":"2022-07-31T03:18:30.700558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train.csv\")\n# test_df = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/test.csv\") useless ,sample not exist","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:18:30.704654Z","iopub.execute_input":"2022-07-31T03:18:30.705511Z","iopub.status.idle":"2022-07-31T03:18:30.727525Z","shell.execute_reply.started":"2022-07-31T03:18:30.705443Z","shell.execute_reply":"2022-07-31T03:18:30.726217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df #guess it's a multi-label classify","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:18:30.729673Z","iopub.execute_input":"2022-07-31T03:18:30.73027Z","iopub.status.idle":"2022-07-31T03:18:30.910507Z","shell.execute_reply.started":"2022-07-31T03:18:30.730231Z","shell.execute_reply":"2022-07-31T03:18:30.909178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path):\n    img=dicom.dcmread(path)\n    data=img.pixel_array\n    data=data-np.min(data)\n    if np.max(data) != 0:\n        data=data/np.max(data)\n    data=(data*255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:18:30.91429Z","iopub.execute_input":"2022-07-31T03:18:30.914884Z","iopub.status.idle":"2022-07-31T03:18:30.923341Z","shell.execute_reply.started":"2022-07-31T03:18:30.914854Z","shell.execute_reply":"2022-07-31T03:18:30.922139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def listdirs(folder):\n    return [d for d in os.listdir(folder) if os.path.isdir(os.path.join(folder, d))]","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:18:30.925879Z","iopub.execute_input":"2022-07-31T03:18:30.926515Z","iopub.status.idle":"2022-07-31T03:18:30.934413Z","shell.execute_reply.started":"2022-07-31T03:18:30.926453Z","shell.execute_reply":"2022-07-31T03:18:30.932737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir='../input/rsna-2022-cervical-spine-fracture-detection/train_images'\npatients = sorted(os.listdir(train_dir))\npatients[:5]","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:18:30.936196Z","iopub.execute_input":"2022-07-31T03:18:30.936916Z","iopub.status.idle":"2022-07-31T03:18:31.046678Z","shell.execute_reply.started":"2022-07-31T03:18:30.936873Z","shell.execute_reply":"2022-07-31T03:18:31.045571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_file = glob.glob(\"../input/rsna-2022-cervical-spine-fracture-detection/train_images/1.2.826.0.1.3680043.10001/*.dcm\")\nplt.figure(figsize=(20, 20))\n\nfor i in range(28):\n    ax = plt.subplot(7, 7, i + 1)\n    # specify your dcm image path\n    image_path = image_file[i]\n\n    image = load_dicom(image_path)\n\n    plt.axis('off')\n    \n    plt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:18:31.048267Z","iopub.execute_input":"2022-07-31T03:18:31.049021Z","iopub.status.idle":"2022-07-31T03:18:33.478104Z","shell.execute_reply.started":"2022-07-31T03:18:31.048917Z","shell.execute_reply":"2022-07-31T03:18:33.477028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nibabel as nib\n\nimage_file = glob.glob(\"../input/rsna-2022-cervical-spine-fracture-detection/segmentations/*.nii\")\nplt.figure(figsize=(20, 20))\n\nfor i in range(28):\n    ax = plt.subplot(7, 7, i + 1)\n    # specify your nii image path\n    image_path = image_file[i]\n    nii_img = nib.load(image_path).get_fdata()\n    nib_image = nii_img[:,:,59]\n    plt.axis('off')\n    plt.imshow(nib_image)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:18:33.479224Z","iopub.execute_input":"2022-07-31T03:18:33.479582Z","iopub.status.idle":"2022-07-31T03:19:16.610355Z","shell.execute_reply.started":"2022-07-31T03:18:33.479546Z","shell.execute_reply":"2022-07-31T03:19:16.609053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Note ❗🏃‍\nwhen use 'dicom.read_file' and apply **'.file_meta.TransferSyntaxUID.name'** you can find 3 types of data:\n\nImplicit VR Little Endian\n\nExplicit VR Little Endian\n\nJPEG Lossless, Non-Hierarchical, First-Order Prediction\n\nThe last one requires jpeg related tools, not yet know how to deal with it. Remove it now otherwise it will stuck the notebook.\n","metadata":{}},{"cell_type":"code","source":"from pydicom.data import get_testdata_files\ntrainset=[]\ntrainlabel=[]\ntrainidt=[]\nlimit = 200 \nfor i in tqdm(range(len(train_df))): #there are 2019 rows, need much times, so just process 10 rows to test\n    idt=train_df.loc[i,'StudyInstanceUID']\n    \n#     idt2=('00000'+str(idt))[-5:]\n    path=os.path.join(train_dir,idt)   \n    \n    for im in os.listdir(path):\n        \n        \n        dc = dicom.read_file(os.path.join(path,im))\n        if dc.file_meta.TransferSyntaxUID.name =='JPEG Lossless, Non-Hierarchical, First-Order Prediction (Process 14 [Selection Value 1])':\n            continue\n        \n        img=load_dicom(os.path.join(path,im)) \n#         ds = decode(os.path.join(path,im))\n#         print(ds)\n#         ds.decompress(\"pylibjpeg\")\n\n        img=cv.resize(img,(64,64)) \n        image=img_to_array(img)\n        image=image/255.0\n\n        trainset+=[image]\n        cur_label=[]\n        cur_label.append(train_df.loc[i, 'patient_overall'])\n        cur_label.append(train_df.loc[i,'C1'])\n        cur_label.append(train_df.loc[i,'C2'])\n        cur_label.append(train_df.loc[i,'C3'])\n        cur_label.append(train_df.loc[i,'C4'])\n        cur_label.append(train_df.loc[i,'C5'])\n        cur_label.append(train_df.loc[i,'C6'])\n        cur_label.append(train_df.loc[i,'C7'])\n        trainlabel+=[cur_label]\n        trainidt+=[idt]\n    i+=1\n    if i==limit:\n        break","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:31:22.549454Z","iopub.execute_input":"2022-07-31T03:31:22.54992Z","iopub.status.idle":"2022-07-31T03:31:43.009511Z","shell.execute_reply.started":"2022-07-31T03:31:22.549878Z","shell.execute_reply":"2022-07-31T03:31:43.007157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=np.array(trainlabel)\nY_train=y\nX_train=np.array(trainset)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:31:54.225Z","iopub.execute_input":"2022-07-31T03:31:54.22544Z","iopub.status.idle":"2022-07-31T03:31:54.292625Z","shell.execute_reply.started":"2022-07-31T03:31:54.225411Z","shell.execute_reply":"2022-07-31T03:31:54.291286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntest_df = ['1.2.826.0.1.3680043.22327', '1.2.826.0.1.3680043.25399', '1.2.826.0.1.3680043.5876']","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:31:57.678293Z","iopub.execute_input":"2022-07-31T03:31:57.67905Z","iopub.status.idle":"2022-07-31T03:31:57.684615Z","shell.execute_reply.started":"2022-07-31T03:31:57.679016Z","shell.execute_reply":"2022-07-31T03:31:57.683189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dir='../input/rsna-2022-cervical-spine-fracture-detection/test_images'\ntestset=[]\ntestidt=[]\nfor i in tqdm(range(len(test_df))):\n    idt=test_df[i]\n    path=os.path.join(test_dir,idt)   \n    \n    for im in os.listdir(path):\n        dc = dicom.read_file(os.path.join(path,im))\n        \n        if dc.file_meta.TransferSyntaxUID.name =='JPEG Lossless, Non-Hierarchical, First-Order Prediction (Process 14 [Selection Value 1])':\n            continue\n        img=load_dicom(os.path.join(path,im)) \n\n        img=cv.resize(img,(64,64)) \n        image=img_to_array(img)\n        image=image/255.0\n        testset+=[image]\n        testidt+=[idt]","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:20:11.762829Z","iopub.execute_input":"2022-07-31T03:20:11.763754Z","iopub.status.idle":"2022-07-31T03:20:32.567296Z","shell.execute_reply.started":"2022-07-31T03:20:11.763668Z","shell.execute_reply":"2022-07-31T03:20:32.565857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = np.array(testset)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:20:32.569308Z","iopub.execute_input":"2022-07-31T03:20:32.570118Z","iopub.status.idle":"2022-07-31T03:20:32.584082Z","shell.execute_reply.started":"2022-07-31T03:20:32.57007Z","shell.execute_reply":"2022-07-31T03:20:32.582727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.models.Sequential()\nmodel.add(keras.layers.Conv2D(filters=64,kernel_size=(4,4),input_shape=(64,64,1),activation='relu',kernel_initializer=\"he_normal\"))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2,2)))\nmodel.add(keras.layers.BatchNormalization())\nmodel.add(keras.layers.Conv2D(filters=64,kernel_size=(4,4),activation='relu',kernel_initializer=\"he_normal\"))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2,2)))\nmodel.add(keras.layers.Dropout(0.20))\nmodel.add(keras.layers.BatchNormalization())\nmodel.add(keras.layers.Conv2D(filters=64,kernel_size=(4,4),activation='relu',kernel_initializer=\"he_normal\"))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2,2)))\nmodel.add(keras.layers.Dropout(0.25))\nmodel.add(keras.layers.BatchNormalization())\nmodel.add(keras.layers.Flatten())\nmodel.add(keras.layers.Dense(100,activation=\"relu\",kernel_initializer=\"he_normal\"))\nmodel.add(keras.layers.Dense(8,\"softmax\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:32:11.8315Z","iopub.execute_input":"2022-07-31T03:32:11.831902Z","iopub.status.idle":"2022-07-31T03:32:11.951561Z","shell.execute_reply.started":"2022-07-31T03:32:11.831871Z","shell.execute_reply":"2022-07-31T03:32:11.95029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, show_shapes=True, rankdir='TB')","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:32:21.980173Z","iopub.execute_input":"2022-07-31T03:32:21.980639Z","iopub.status.idle":"2022-07-31T03:32:22.25256Z","shell.execute_reply.started":"2022-07-31T03:32:21.98061Z","shell.execute_reply":"2022-07-31T03:32:22.25097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss=\"binary_crossentropy\",\n              optimizer = \"RMSprop\",metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:32:26.447997Z","iopub.execute_input":"2022-07-31T03:32:26.448389Z","iopub.status.idle":"2022-07-31T03:32:26.464087Z","shell.execute_reply.started":"2022-07-31T03:32:26.448356Z","shell.execute_reply":"2022-07-31T03:32:26.462864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callback = keras.callbacks.EarlyStopping(monitor='loss', patience=8)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:32:30.146642Z","iopub.execute_input":"2022-07-31T03:32:30.147019Z","iopub.status.idle":"2022-07-31T03:32:30.153269Z","shell.execute_reply.started":"2022-07-31T03:32:30.14699Z","shell.execute_reply":"2022-07-31T03:32:30.152027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist = model.fit(X_train, Y_train,epochs=100, batch_size=64, verbose=1,callbacks=[callback])","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:32:34.469767Z","iopub.execute_input":"2022-07-31T03:32:34.470342Z","iopub.status.idle":"2022-07-31T03:33:02.991978Z","shell.execute_reply.started":"2022-07-31T03:32:34.470302Z","shell.execute_reply":"2022-07-31T03:33:02.990643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=model.predict(X_test)\n\n# pred=np.argmax(y_pred,axis=1)\n# pred[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:34:57.974748Z","iopub.execute_input":"2022-07-31T03:34:57.975601Z","iopub.status.idle":"2022-07-31T03:34:58.093435Z","shell.execute_reply.started":"2022-07-31T03:34:57.975556Z","shell.execute_reply":"2022-07-31T03:34:58.092151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.DataFrame(columns = train_df.columns, index = range(len(testidt)))","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:47:36.981461Z","iopub.execute_input":"2022-07-31T03:47:36.981857Z","iopub.status.idle":"2022-07-31T03:47:36.990511Z","shell.execute_reply.started":"2022-07-31T03:47:36.981829Z","shell.execute_reply":"2022-07-31T03:47:36.989159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(range(len(testidt))):\n    result.loc[i, 'StudyInstanceUID'] = testidt[i]\n    rows = np.int64(y_pred[i]>0.07)\n    result.loc[i, 'patient_overall'] = rows[0]\n    result.loc[i, 'C1'] = rows[1]\n    result.loc[i, 'C2'] = rows[2]\n    result.loc[i, 'C3'] = rows[3]\n    result.loc[i, 'C4'] = rows[4]\n    result.loc[i, 'C5'] = rows[5]\n    result.loc[i, 'C6'] = rows[6]\n    result.loc[i, 'C7'] = rows[7]","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:50:54.114811Z","iopub.execute_input":"2022-07-31T03:50:54.115208Z","iopub.status.idle":"2022-07-31T03:50:55.083567Z","shell.execute_reply.started":"2022-07-31T03:50:54.115179Z","shell.execute_reply":"2022-07-31T03:50:55.082118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"means = result[['patient_overall', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7']].mean().to_dict()","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:54:21.760959Z","iopub.execute_input":"2022-07-31T03:54:21.761367Z","iopub.status.idle":"2022-07-31T03:54:21.778187Z","shell.execute_reply.started":"2022-07-31T03:54:21.761336Z","shell.execute_reply":"2022-07-31T03:54:21.77655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntest_df2 = pd.read_csv('../input/rsna-2022-cervical-spine-fracture-detection/test.csv')\n# means = train_df.mean(numeric_only=True).to_dict()\ntest_df2['fractured'] = test_df2['prediction_type'].map(means)\n\ntest_df2[['row_id','fractured']].to_csv('submission.csv', index=False, float_format='%.1g')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-31T03:56:05.034598Z","iopub.execute_input":"2022-07-31T03:56:05.03576Z","iopub.status.idle":"2022-07-31T03:56:05.05036Z","shell.execute_reply.started":"2022-07-31T03:56:05.035728Z","shell.execute_reply":"2022-07-31T03:56:05.048684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Actual Work 🤔❓\n\nHow to deal with jpeg image?\n\nHow to make submission correcct?\n\nHow to use the segmentation to help predict?","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}