{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA Knee Abnormality Detection - EDA\n\nThis notebook builds a practical exploratory data analysis workflow for the Kaggle competition dataset.\n\nIt covers:\n- study-level label and report analysis (`train.csv`)\n- series-level protocol analysis (`train_series.csv`)\n- sampled DICOM metadata and pixel checks (`train_series/`)\n- quick checks to validate modeling readiness","metadata":{}},{"cell_type":"code","source":"from __future__ import annotations\n\nfrom pathlib import Path\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.set_theme(style=\"whitegrid\")\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_colwidth\", 120)\n\nRANDOM_SEED = 42\nrandom.seed(RANDOM_SEED)\nnp.random.seed(RANDOM_SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:18:02.95227Z","iopub.execute_input":"2026-08-05T18:18:02.952672Z","iopub.status.idle":"2026-08-05T18:18:04.1171Z","shell.execute_reply.started":"2026-08-05T18:18:02.952637Z","shell.execute_reply":"2026-08-05T18:18:04.116445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def resolve_data_root() -> Path:\n    \"\"\"Finds dataset root in Kaggle or local workspace.\"\"\"\n    candidates = [\n        Path(\"/kaggle/input/competitions/rsna-knee-abnormality-detection\"),\n        Path.cwd(),\n        Path.cwd() / \"data\",\n    ]\n    for candidate in candidates:\n        if (candidate / \"train.csv\").exists() or (candidate / \"sample_submission.csv\").exists():\n            return candidate\n    return Path.cwd()\n\ndata_root = resolve_data_root()\nprint(f\"Data root: {data_root.resolve()}\")\nfor name in [\"train.csv\", \"train_series.csv\", \"test.csv\", \"test_series.csv\", \"sample_submission.csv\", \"train_series\"]:\n    print(f\"{name:20s} -> {(data_root / name).exists()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:19:44.671086Z","iopub.execute_input":"2026-08-05T18:19:44.671441Z","iopub.status.idle":"2026-08-05T18:19:44.68554Z","shell.execute_reply.started":"2026-08-05T18:19:44.671418Z","shell.execute_reply":"2026-08-05T18:19:44.684815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_csv_if_exists(path: Path) -> pd.DataFrame:\n    if path.exists():\n        return pd.read_csv(path)\n    print(f\"[WARN] Missing: {path}\")\n    return pd.DataFrame()\n\n\ntrain_df = read_csv_if_exists(data_root / \"train.csv\")\ntrain_series_df = read_csv_if_exists(data_root / \"train_series.csv\")\ntest_df = read_csv_if_exists(data_root / \"test.csv\")\ntest_series_df = read_csv_if_exists(data_root / \"test_series.csv\")\nsubmission_df = read_csv_if_exists(data_root / \"sample_submission.csv\")\n\nprint(\"\\nShapes\")\nfor name, df in [\n    (\"train\", train_df),\n    (\"train_series\", train_series_df),\n    (\"test\", test_df),\n    (\"test_series\", test_series_df),\n    (\"sample_submission\", submission_df),\n]:\n    print(f\"{name:20s} {df.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:20:19.871332Z","iopub.execute_input":"2026-08-05T18:20:19.871715Z","iopub.status.idle":"2026-08-05T18:20:20.138608Z","shell.execute_reply.started":"2026-08-05T18:20:19.871692Z","shell.execute_reply":"2026-08-05T18:20:20.137914Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1) Study-Level Overview","metadata":{"execution":{"iopub.status.busy":"2026-08-05T18:20:33.948983Z","iopub.execute_input":"2026-08-05T18:20:33.94932Z","iopub.status.idle":"2026-08-05T18:20:33.953632Z","shell.execute_reply.started":"2026-08-05T18:20:33.949298Z","shell.execute_reply":"2026-08-05T18:20:33.952741Z"}}},{"cell_type":"code","source":"train_df.","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not train_df.empty:\n    display(train_df.head(3))\n\n    print(\"Unique studies:\", train_df[\"StudyInstanceUID\"].nunique() if \"StudyInstanceUID\" in train_df else \"N/A\")\n\n    missing_pct = train_df.isna().mean().sort_values(ascending=False) * 100\n    display(missing_pct[missing_pct > 0].to_frame(\"missing_%\").head(20))\nelse:\n    print(\"train.csv not available yet. Download the competition data to run full EDA.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:20:49.500178Z","iopub.execute_input":"2026-08-05T18:20:49.500482Z","iopub.status.idle":"2026-08-05T18:20:49.542829Z","shell.execute_reply.started":"2026-08-05T18:20:49.500462Z","shell.execute_reply":"2026-08-05T18:20:49.542025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"EXPECTED_LABELS = [\n    \"ACL\", \"MCL\", \"Medial Meniscus\", \"Lateral Meniscus\",\n    \"Medial OA\", \"Lateral OA\", \"PF OA\", \"Effusion\",\n    \"Synovitis\", \"Baker's\", \"Contusion\", \"Fracture\",\n]\n\nlabel_cols = [c for c in EXPECTED_LABELS if c in train_df.columns]\nprint(\"Detected label columns:\", label_cols)\n\nif label_cols:\n    prevalence = train_df[label_cols].mean().sort_values(ascending=False)\n\n    plt.figure(figsize=(10, 5))\n    sns.barplot(x=prevalence.index, y=prevalence.values, palette=\"viridis\")\n    plt.xticks(rotation=45, ha=\"right\")\n    plt.ylabel(\"Positive rate\")\n    plt.title(\"Label prevalence in train.csv\")\n    plt.tight_layout()\n    plt.show()\n\n    display((prevalence * 100).round(2).to_frame(\"positive_%\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:29:37.144161Z","iopub.execute_input":"2026-08-05T18:29:37.144475Z","iopub.status.idle":"2026-08-05T18:29:37.366973Z","shell.execute_reply.started":"2026-08-05T18:29:37.144454Z","shell.execute_reply":"2026-08-05T18:29:37.36631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if label_cols:\n    corr = train_df[label_cols].corr()\n\n    plt.figure(figsize=(9, 7))\n    sns.heatmap(corr, cmap=\"coolwarm\", center=0, vmin=-1, vmax=1, square=True)\n    plt.title(\"Label correlation matrix\")\n    plt.tight_layout()\n    plt.show()\n\n    positive_labels_per_study = train_df[label_cols].sum(axis=1)\n    plt.figure(figsize=(8, 4))\n    sns.countplot(x=positive_labels_per_study, color=\"steelblue\")\n    plt.title(\"Number of positive findings per study\")\n    plt.xlabel(\"Positive label count\")\n    plt.ylabel(\"Studies\")\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:21:58.037744Z","iopub.execute_input":"2026-08-05T18:21:58.038065Z","iopub.status.idle":"2026-08-05T18:21:58.532372Z","shell.execute_reply.started":"2026-08-05T18:21:58.038037Z","shell.execute_reply":"2026-08-05T18:21:58.531414Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2) Series-Level Protocol Analysis","metadata":{"execution":{"iopub.status.busy":"2026-08-05T18:30:01.33339Z","iopub.execute_input":"2026-08-05T18:30:01.333733Z","iopub.status.idle":"2026-08-05T18:30:01.338368Z","shell.execute_reply.started":"2026-08-05T18:30:01.333709Z","shell.execute_reply":"2026-08-05T18:30:01.337397Z"}}},{"cell_type":"code","source":"if not train_series_df.empty:\n    display(train_series_df.head(3))\n\n    if \"StudyInstanceUID\" in train_series_df:\n        series_per_study = train_series_df.groupby(\"StudyInstanceUID\").size()\n        plt.figure(figsize=(8, 4))\n        sns.histplot(series_per_study, bins=30, kde=True)\n        plt.title(\"Series per study (train)\")\n        plt.xlabel(\"Number of series\")\n        plt.tight_layout()\n        plt.show()\n\n        display(series_per_study.describe().to_frame(\"series_per_study\"))\n\n    for col in [\"Anatomical_Plane\", \"Fluid_Sensitive\", \"Fat_Suppression\"]:\n        if col in train_series_df:\n            plt.figure(figsize=(7, 4))\n            vc = train_series_df[col].astype(str).value_counts()\n            sns.barplot(x=vc.index, y=vc.values, palette=\"mako\")\n            plt.title(f\"{col} distribution\")\n            plt.ylabel(\"Count\")\n            plt.tight_layout()\n            plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:30:31.044893Z","iopub.execute_input":"2026-08-05T18:30:31.045186Z","iopub.status.idle":"2026-08-05T18:30:31.652502Z","shell.execute_reply.started":"2026-08-05T18:30:31.045165Z","shell.execute_reply":"2026-08-05T18:30:31.651919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if (not train_df.empty) and (not train_series_df.empty) and label_cols:\n    merged = train_series_df.merge(\n        train_df[[\"StudyInstanceUID\", *label_cols]],\n        on=\"StudyInstanceUID\",\n        how=\"left\",\n    )\n\n    if \"Anatomical_Plane\" in merged:\n        plane_label_pos = merged.groupby(\"Anatomical_Plane\")[label_cols].mean().T\n        plt.figure(figsize=(10, 5))\n        sns.heatmap(plane_label_pos, cmap=\"YlGnBu\", annot=False)\n        plt.title(\"Positive label rates by anatomical plane\")\n        plt.tight_layout()\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:31:01.618595Z","iopub.execute_input":"2026-08-05T18:31:01.618972Z","iopub.status.idle":"2026-08-05T18:31:01.835995Z","shell.execute_reply.started":"2026-08-05T18:31:01.618939Z","shell.execute_reply":"2026-08-05T18:31:01.83513Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3) DICOM-Level Checks (Sampled)\n\nThis section samples a limited number of DICOM slices so EDA stays fast.","metadata":{"execution":{"iopub.status.busy":"2026-08-05T18:31:14.832162Z","iopub.execute_input":"2026-08-05T18:31:14.832942Z","iopub.status.idle":"2026-08-05T18:31:14.839167Z","shell.execute_reply.started":"2026-08-05T18:31:14.83291Z","shell.execute_reply":"2026-08-05T18:31:14.837773Z"}}},{"cell_type":"code","source":"try:\n    import pydicom\nexcept ImportError:\n    raise ImportError(\"Please install pydicom: pip install pydicom\")\n\ntrain_series_dir = data_root / \"train_series\"\nif train_series_dir.exists():\n    dcm_paths = list(train_series_dir.rglob(\"*.dcm\"))\n    print(f\"Total DICOM files discovered: {len(dcm_paths):,}\")\nelse:\n    dcm_paths = []\n    print(\"train_series directory not found.\")\n\nSAMPLE_DCM = 400\nsampled_dcm_paths = random.sample(dcm_paths, min(SAMPLE_DCM, len(dcm_paths))) if dcm_paths else []\nprint(f\"Sampled {len(sampled_dcm_paths)} DICOM files\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:31:30.480862Z","iopub.execute_input":"2026-08-05T18:31:30.481258Z","iopub.status.idle":"2026-08-05T18:39:57.058741Z","shell.execute_reply.started":"2026-08-05T18:31:30.481225Z","shell.execute_reply":"2026-08-05T18:39:57.058103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meta_rows = []\nfor path in sampled_dcm_paths:\n    try:\n        ds = pydicom.dcmread(path, stop_before_pixels=True, force=True)\n        meta_rows.append({\n            \"path\": str(path),\n            \"Rows\": getattr(ds, \"Rows\", np.nan),\n            \"Columns\": getattr(ds, \"Columns\", np.nan),\n            \"BitsStored\": getattr(ds, \"BitsStored\", np.nan),\n            \"PixelRepresentation\": getattr(ds, \"PixelRepresentation\", np.nan),\n            \"PhotometricInterpretation\": getattr(ds, \"PhotometricInterpretation\", None),\n            \"TransferSyntaxUID\": str(getattr(getattr(ds, \"file_meta\", None), \"TransferSyntaxUID\", \"Unknown\")),\n            \"SliceThickness\": getattr(ds, \"SliceThickness\", np.nan),\n        })\n    except Exception:\n        continue\n\nmeta_df = pd.DataFrame(meta_rows)\nprint(\"Metadata rows:\", len(meta_df))\ndisplay(meta_df.head(3))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:58:37.903262Z","iopub.execute_input":"2026-08-05T18:58:37.903656Z","iopub.status.idle":"2026-08-05T18:58:42.730854Z","shell.execute_reply.started":"2026-08-05T18:58:37.903616Z","shell.execute_reply":"2026-08-05T18:58:42.730218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not meta_df.empty:\n    fig, axes = plt.subplots(1, 3, figsize=(14, 4))\n\n    sns.histplot(meta_df[\"Rows\"].dropna(), bins=30, ax=axes[0], color=\"teal\")\n    axes[0].set_title(\"Rows\")\n\n    sns.histplot(meta_df[\"Columns\"].dropna(), bins=30, ax=axes[1], color=\"slateblue\")\n    axes[1].set_title(\"Columns\")\n\n    sns.histplot(meta_df[\"SliceThickness\"].dropna(), bins=30, ax=axes[2], color=\"tomato\")\n    axes[2].set_title(\"Slice Thickness\")\n\n    plt.tight_layout()\n    plt.show()\n\n    plt.figure(figsize=(10, 4))\n    ts_counts = meta_df[\"TransferSyntaxUID\"].value_counts().head(10)\n    sns.barplot(x=ts_counts.index, y=ts_counts.values, palette=\"crest\")\n    plt.xticks(rotation=45, ha=\"right\")\n    plt.title(\"Most common Transfer Syntax UIDs (sample)\")\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:58:51.523327Z","iopub.execute_input":"2026-08-05T18:58:51.523763Z","iopub.status.idle":"2026-08-05T18:58:52.084911Z","shell.execute_reply.started":"2026-08-05T18:58:51.523737Z","shell.execute_reply":"2026-08-05T18:58:52.083998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_normalized_pixel_array(path: Path) -> np.ndarray:\n    ds = pydicom.dcmread(path, force=True)\n    arr = ds.pixel_array.astype(np.float32)\n    arr = (arr - arr.min()) / (arr.max() - arr.min() + 1e-6)\n    return arr\n\n\nif sampled_dcm_paths:\n    preview_paths = random.sample(sampled_dcm_paths, min(6, len(sampled_dcm_paths)))\n    fig, axes = plt.subplots(2, 3, figsize=(12, 8))\n\n    for ax, p in zip(axes.flat, preview_paths):\n        try:\n            img = load_normalized_pixel_array(p)\n            ax.imshow(img, cmap=\"gray\")\n            ax.set_title(p.parent.name[:10])\n            ax.axis(\"off\")\n        except Exception:\n            ax.axis(\"off\")\n            ax.set_title(\"Read error\")\n\n    plt.suptitle(\"Random DICOM slice previews\")\n    plt.tight_layout()\n    plt.show()\nelse:\n    print(\"No DICOM files available for preview.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:59:01.924488Z","iopub.execute_input":"2026-08-05T18:59:01.9249Z","iopub.status.idle":"2026-08-05T18:59:02.548488Z","shell.execute_reply.started":"2026-08-05T18:59:01.924875Z","shell.execute_reply":"2026-08-05T18:59:02.547598Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4) Report Text Quick Profiling","metadata":{}},{"cell_type":"code","source":"if \"Report\" in train_df:\n    report_series = train_df[\"Report\"].fillna(\"\").astype(str)\n    report_len = report_series.str.len()\n\n    plt.figure(figsize=(9, 4))\n    sns.histplot(report_len, bins=50, kde=True, color=\"purple\")\n    plt.title(\"Report character-length distribution\")\n    plt.xlabel(\"Characters\")\n    plt.tight_layout()\n    plt.show()\n\n    # Simple heuristic: fraction of ascii characters as a rough language diversity signal.\n    def ascii_fraction(text: str) -> float:\n        if not text:\n            return np.nan\n        ascii_chars = sum(1 for ch in text if ord(ch) < 128)\n        return ascii_chars / len(text)\n\n    ascii_frac = report_series.sample(min(5000, len(report_series)), random_state=RANDOM_SEED).map(ascii_fraction)\n    plt.figure(figsize=(8, 4))\n    sns.histplot(ascii_frac.dropna(), bins=40, color=\"darkorange\")\n    plt.title(\"ASCII character fraction in reports (sample)\")\n    plt.xlabel(\"ASCII fraction\")\n    plt.tight_layout()\n    plt.show()\n\n    display(report_series.sample(min(5, len(report_series)), random_state=RANDOM_SEED))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-05T18:59:14.33566Z","iopub.execute_input":"2026-08-05T18:59:14.336025Z","iopub.status.idle":"2026-08-05T18:59:14.917786Z","shell.execute_reply.started":"2026-08-05T18:59:14.335992Z","shell.execute_reply":"2026-08-05T18:59:14.917177Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5) Next Modeling-Oriented Checks\n\nRecommended follow-up analyses:\n- split by study/site proxies and verify label prevalence drift\n- inspect orientation and spacing consistency per `Anatomical_Plane`\n- build per-series sampling strategy for training (plane + fluid/fat balance)\n- evaluate report-derived weak labels for unlabeled studies","metadata":{}}]}