{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.9","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594},{"sourceType":"datasetVersion","sourceId":1832961,"datasetId":1089494,"databundleVersionId":1870583},{"sourceType":"datasetVersion","sourceId":2733224,"datasetId":1125146,"databundleVersionId":2778331}],"dockerImageVersionId":30085,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib\nimport pydicom as dicom\nimport cv2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T15:48:16.435095Z","iopub.execute_input":"2024-12-11T15:48:16.435498Z","iopub.status.idle":"2024-12-11T15:48:16.873677Z","shell.execute_reply.started":"2024-12-11T15:48:16.435414Z","shell.execute_reply":"2024-12-11T15:48:16.872434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = pd.read_csv(path+'train.csv')\nsamp_subm = pd.read_csv(path+'sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T15:48:16.889431Z","iopub.execute_input":"2024-12-11T15:48:16.889747Z","iopub.status.idle":"2024-12-11T15:48:17.094907Z","shell.execute_reply.started":"2024-12-11T15:48:16.889717Z","shell.execute_reply":"2024-12-11T15:48:17.093754Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**5. Exploratory Data Analysis**","metadata":{}},{"cell_type":"markdown","source":"As we can see the dataset is inbalanced.","metadata":{}},{"cell_type":"code","source":"# Read DICOM Files\nidnum = 2\nimage_id = train_data.loc[idnum, 'image_id']\ndata_file = dicom.dcmread(path+'train/'+image_id+'.dicom')\nimg = data_file.pixel_array","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T15:48:17.40836Z","iopub.execute_input":"2024-12-11T15:48:17.408658Z","iopub.status.idle":"2024-12-11T15:48:19.726252Z","shell.execute_reply.started":"2024-12-11T15:48:17.408622Z","shell.execute_reply":"2024-12-11T15:48:19.725101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(data_file)","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T15:48:19.728013Z","iopub.execute_input":"2024-12-11T15:48:19.728336Z","iopub.status.idle":"2024-12-11T15:48:19.735836Z","shell.execute_reply.started":"2024-12-11T15:48:19.728299Z","shell.execute_reply":"2024-12-11T15:48:19.734202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Image shape:', img.shape)","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T15:48:19.737709Z","iopub.execute_input":"2024-12-11T15:48:19.738194Z","iopub.status.idle":"2024-12-11T15:48:19.745995Z","shell.execute_reply.started":"2024-12-11T15:48:19.738036Z","shell.execute_reply":"2024-12-11T15:48:19.745031Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bbox = [train_data.loc[idnum, 'x_min'],\n        train_data.loc[idnum, 'y_min'],\n        train_data.loc[idnum, 'x_max'],\n        train_data.loc[idnum, 'y_max']]\nfig, ax = plt.subplots(1, 1, figsize=(20, 4))\nax.imshow(img, cmap='gray')\np = matplotlib.patches.Rectangle((bbox[0], bbox[1]),\n                                 bbox[2]-bbox[0],\n                                 bbox[3]-bbox[1],\n                                 ec='r', fc='none', lw=2.)\nax.add_patch(p)\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T15:48:19.747324Z","iopub.execute_input":"2024-12-11T15:48:19.747743Z","iopub.status.idle":"2024-12-11T15:48:20.407459Z","shell.execute_reply.started":"2024-12-11T15:48:19.747701Z","shell.execute_reply":"2024-12-11T15:48:20.406176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_train_data(idx_list):\n    fig, axs = plt.subplots(1, 3, figsize=(15, 10))\n    fig.subplots_adjust(hspace = .1, wspace=.1)\n    axs = axs.ravel()\n    for i in range(3):\n        image_id = train_data.loc[idx_list[i], 'image_id']\n        data_file = dicom.dcmread(path+'train/'+image_id+'.dicom')\n        img = data_file.pixel_array\n        axs[i].imshow(img, cmap='gray')\n        axs[i].set_title(train_data.loc[idx_list[i], 'class_name'])\n        axs[i].set_xticklabels([])\n        axs[i].set_yticklabels([])\n        if train_data.loc[idx_list[i], 'class_name'] != 'No finding':\n            bbox = [train_data.loc[idx_list[i], 'x_min'],\n                    train_data.loc[idx_list[i], 'y_min'],\n                    train_data.loc[idx_list[i], 'x_max'],\n                    train_data.loc[idx_list[i], 'y_max']]\n            p = matplotlib.patches.Rectangle((bbox[0], bbox[1]),\n                                             bbox[2]-bbox[0],\n                                             bbox[3]-bbox[1],\n                                             ec='r', fc='none', lw=2.)\n            axs[i].add_patch(p)","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T15:48:20.409341Z","iopub.execute_input":"2024-12-11T15:48:20.409763Z","iopub.status.idle":"2024-12-11T15:48:20.420601Z","shell.execute_reply.started":"2024-12-11T15:48:20.409719Z","shell.execute_reply":"2024-12-11T15:48:20.419492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pydicom as dicom\nimport cv2\n\n# Suppress warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Step 1: Define the dataset path\ndataset_path = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/'\n\n# Step 2: List the files in the dataset directory\ndef list_files(dataset_path):\n    \"\"\"Lists files in the given directory.\"\"\"\n    return os.listdir(dataset_path)\n\n# Step 3: Load datasets\ndef load_datasets(dataset_path):\n    \"\"\"Loads train and sample submission datasets.\"\"\"\n    train_data = pd.read_csv(os.path.join(dataset_path, 'train.csv'))\n    samp_subm = pd.read_csv(os.path.join(dataset_path, 'sample_submission.csv'))\n    return train_data, samp_subm\n\n# Step 4: Print dataset statistics\ndef print_data_statistics(train_data, samp_subm):\n    \"\"\"Prints the number of train and test samples.\"\"\"\n    print(f\"Number of training samples: {len(train_data)}\")\n    print(f\"Number of test samples: {len(samp_subm)}\")\n\n# Step 5: Visualize class distribution\ndef visualize_class_distribution(train_data):\n    \"\"\"Visualizes the distribution of classes in the training dataset.\"\"\"\n    fig, ax = plt.subplots(figsize=(12, 4))\n    class_counts = train_data['class_name'].value_counts()\n    ax.bar(class_counts.index, class_counts.values)\n    ax.set_xticklabels(class_counts.index, rotation=90)\n    ax.set_title('Distribution of the labels')\n    plt.grid()\n    plt.show()\n\n# Test case function\ndef test_case():\n    \"\"\"Test case for verifying the workflow.\"\"\"\n    # Simulate dataset path (this path needs to exist for the test to run)\n    test_path = dataset_path\n\n    # Step 1: Check if files are listed\n    files = list_files(test_path)\n    print(\"Files in dataset:\", files)\n\n    # Step 2: Load datasets\n    train_data, samp_subm = load_datasets(test_path)\n\n    # Step 3: Print statistics\n    print_data_statistics(train_data, samp_subm)\n\n    # Step 4: Visualize class distribution\n    visualize_class_distribution(train_data)\n\n# Run the test case\ntest_case()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T16:13:12.830637Z","iopub.execute_input":"2024-12-11T16:13:12.83098Z","iopub.status.idle":"2024-12-11T16:13:13.172637Z","shell.execute_reply.started":"2024-12-11T16:13:12.830951Z","shell.execute_reply":"2024-12-11T16:13:13.171707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for num in range(15):\n    idx_list = train_data[train_data['class_id']==num][0:3].index.values\n    plot_train_data(idx_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T15:48:20.422254Z","iopub.execute_input":"2024-12-11T15:48:20.422547Z","iopub.status.idle":"2024-12-11T15:49:43.803949Z","shell.execute_reply.started":"2024-12-11T15:48:20.422513Z","shell.execute_reply":"2024-12-11T15:49:43.802823Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"6. Results\n","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0"}},{"cell_type":"code","source":"#samp_subm.to_csv('submission1.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T15:49:43.805275Z","iopub.execute_input":"2024-12-11T15:49:43.805543Z","iopub.status.idle":"2024-12-11T15:49:43.80961Z","shell.execute_reply.started":"2024-12-11T15:49:43.805516Z","shell.execute_reply":"2024-12-11T15:49:43.808293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_2class = pd.read_csv(\"../input/vinbigdata-2class-prediction/2-cls test pred.csv\")\nlow_threshold = 0.001\nhigh_threshold = 0.87\npred_2class","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T15:49:43.810948Z","iopub.execute_input":"2024-12-11T15:49:43.811319Z","iopub.status.idle":"2024-12-11T15:49:43.85503Z","shell.execute_reply.started":"2024-12-11T15:49:43.81128Z","shell.execute_reply":"2024-12-11T15:49:43.853993Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**7. Discussion**","metadata":{}},{"cell_type":"markdown","source":"**8. Furture Improvement**\n","metadata":{}},{"cell_type":"markdown","source":"**9. References**\n\n1. https://www.kaggle.com/kyawkyaw/vinbigdata-chest-x-ray-abnormalities-classifier","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}}]}