{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nbase = '/kaggle/input/competitions/rsna-knee-abnormality-detection'\nfor item in os.listdir(base):\n    print(item)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ntrain = pd.read_csv('/kaggle/input/competitions/rsna-knee-abnormality-detection/train.csv')\nprint(train.shape)\ntrain.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"label_cols = ['ACL','MCL','Medial Meniscus','Lateral Meniscus','Medial OA',\n              'Lateral OA','PF OA','Effusion','Synovitis',\"Baker's\",'Contusion','Fracture']\n\nlabeled = train.dropna(subset=label_cols)\nprint(f'Total studies: {len(train)}')\nprint(f'Labeled studies: {len(labeled)}')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# how many labels are actually filled in, per column?\ntrain[label_cols].notna().sum()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series = pd.read_csv('/kaggle/input/competitions/rsna-knee-abnormality-detection/train_series.csv')\nprint(series.shape)\nseries.head()\n\n# how many series belong to our 58 labeled studies?\nlabeled_ids = labeled['StudyInstanceUID'].tolist()\nseries[series['StudyInstanceUID'].isin(labeled_ids)].shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series.head(10)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series['Anatomical_Plane'].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# filter to just Sagittal series belonging to our 58 labeled studies\nsag = series[\n    (series['Anatomical_Plane'] == 'Sagittal') &\n    (series['StudyInstanceUID'].isin(labeled_ids))\n]\nprint(sag.shape)\n\n# some studies might have more than one Sagittal series - just keep the first one per study\none_per_study = sag.drop_duplicates('StudyInstanceUID')\nprint(one_per_study.shape)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"row = one_per_study.iloc[0]\nstudy_id = row['StudyInstanceUID']\nseries_id = row['SeriesInstanceUID']\n\nseries_dir = f'/kaggle/input/competitions/rsna-knee-abnormality-detection/train_series/{study_id}/{series_id}'\n\nimport os\nfiles = sorted(os.listdir(series_dir))\nprint(len(files))\nprint(files[:5])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pydicom\nimport matplotlib.pyplot as plt\n\ndcm = pydicom.dcmread(f'{series_dir}/{files[len(files)//2]}')  # grab the middle slice\narr = dcm.pixel_array\nprint(arr.shape, arr.dtype)\n\nplt.imshow(arr, cmap='gray')\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom PIL import Image\nimport os\n\nout_dir = '/kaggle/working/train_pngs'\nos.makedirs(out_dir, exist_ok=True)\n\nfor _, row in one_per_study.iterrows():\n    study_id = row['StudyInstanceUID']\n    series_id = row['SeriesInstanceUID']\n    series_dir = f'/kaggle/input/competitions/rsna-knee-abnormality-detection/train_series/{study_id}/{series_id}'\n    \n    files = sorted(os.listdir(series_dir))\n    mid_file = files[len(files)//2]\n    \n    dcm = pydicom.dcmread(f'{series_dir}/{mid_file}')\n    arr = dcm.pixel_array.astype(np.float32)\n    \n    # normalize to 0-255\n    arr = (arr - arr.min()) / (arr.max() - arr.min() + 1e-6) * 255\n    arr = arr.astype(np.uint8)\n    \n    Image.fromarray(arr).save(f'{out_dir}/{study_id}.png')\n\nprint(len(os.listdir(out_dir)))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.vision.all import *\n\n# build a dataframe fastai can read: filename + the 12 labels\ndf = labeled.copy()\ndf['fname'] = df['StudyInstanceUID'] + '.png'\n\ndls = ImageDataLoaders.from_df(\n    df,\n    path=out_dir,\n    fn_col='fname',\n    label_col=label_cols,\n    y_block=MultiCategoryBlock(encoded=True, vocab=label_cols),\n    item_tfms=Resize(224),\n    valid_pct=0.2,\n    bs=8,\n    seed=42\n)\n\ndls.show_batch()\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# grab a few unlabeled studies' reports\nunlabeled = train[train[label_cols].isna().all(axis=1)]\nprint(len(unlabeled))\n\nfor r in unlabeled['Report'].head(5):\n    print(r)\n    print('---')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# rough language check using a simple heuristic first\nsample = unlabeled['Report'].head(200)\nenglish_like = sample.str.contains(r'\\b(the|and|is|normal|meniscus)\\b', case=False, na=False)\nprint(english_like.sum(), '/', len(sample))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from langdetect import detect, LangDetectException\n\ndef is_english(text):\n    try:\n        return detect(text) == 'en'\n    except LangDetectException:\n        return False\n\n# test on a sample first, this can be slow on 4000+ texts\nsample_reports = unlabeled['Report'].head(200)\nsample_is_en = sample_reports.apply(is_english)\nprint(sample_is_en.sum(), '/', len(sample_reports))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\ntqdm.pandas()\nunlabeled = train[train[label_cols].isna().all(axis=1)].copy()\nunlabeled['is_english'] = unlabeled['Report'].progress_apply(is_english)\nenglish_reports = unlabeled[unlabeled['is_english']]\nprint(len(english_reports))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef check_finding(report, positive_terms, negation_words=None, window=6):\n    if negation_words is None:\n        negation_words = ['no', 'not', 'without', 'absent', 'negative', 'none']\n    text = report.lower()\n    words = re.findall(r'\\w+', text)\n    for term in positive_terms:\n        for i, w in enumerate(words):\n            if term in w:\n                context = words[max(0, i-window):i]\n                if any(neg in context for neg in negation_words):\n                    return 0\n                return 1\n    return None  # not mentioned -> leave unlabeled for this condition\n\nenglish_reports = english_reports.copy()\nenglish_reports['Fracture_extracted'] = english_reports['Report'].apply(\n    lambda r: check_finding(r, ['fracture'])\n)\n\n# spot check\nsample_check = english_reports[english_reports['Fracture_extracted'].notna()].head(15)\nfor _, row in sample_check.iterrows():\n    print(row['Fracture_extracted'], '|', row['Report'][:150])\n    print('---')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# term lists per condition - start rough, we'll refine based on spot checks\nterm_map = {\n    'ACL': ['acl', 'anterior cruciate'],\n    'MCL': ['mcl', 'medial collateral'],\n    'Medial Meniscus': ['medial meniscus'],\n    'Lateral Meniscus': ['lateral meniscus'],\n    'Medial OA': ['medial compartment osteoarthritis', 'medial osteoarthritis', 'medial oa'],\n    'Lateral OA': ['lateral compartment osteoarthritis', 'lateral osteoarthritis', 'lateral oa'],\n    'PF OA': ['patellofemoral osteoarthritis', 'patellofemoral oa'],\n    'Effusion': ['effusion'],\n    'Synovitis': ['synovitis'],\n    \"Baker's\": [\"baker\", \"popliteal cyst\"],\n    'Contusion': ['contusion', 'bone bruise'],\n    'Fracture': ['fracture'],\n}\n\nfor label, terms in term_map.items():\n    english_reports[f'{label}_extracted'] = english_reports['Report'].apply(\n        lambda r: check_finding(r, terms)\n    )","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def check_finding(report, positive_terms, negation_words=None, window_chars=60):\n    if negation_words is None:\n        negation_words = ['no', 'not', 'without', 'absent', 'negative', 'none',\n                           'intact', 'normal', 'unremarkable', 'no evidence']\n    text = report.lower()\n    for term in positive_terms:\n        idx = text.find(term)\n        if idx == -1:\n            continue\n        before = text[max(0, idx-window_chars):idx]\n        after = text[idx+len(term):idx+len(term)+window_chars]\n        if any(neg in before for neg in negation_words) or any(neg in after for neg in negation_words):\n            return 0\n        return 1\n    return None\n\nfor label, terms in term_map.items():\n    english_reports[f'{label}_extracted'] = english_reports['Report'].apply(\n        lambda r: check_finding(r, terms)\n    )\n\nfor label in term_map:\n    col = f'{label}_extracted'\n    n_found = english_reports[col].notna().sum()\n    n_pos = english_reports[col].sum()\n    print(f'{label}: {n_found} labeled ({n_pos} positive)')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = english_reports[english_reports['ACL_extracted'].notna()].sample(15, random_state=1)\nfor _, row in sample.iterrows():\n    print(row['ACL_extracted'], '|', row['Report'][:200])\n    print('---')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"extracted_cols = [f'{label}_extracted' for label in label_cols]\n\n# rename extracted columns to match original label names\nauto_labels = english_reports[['StudyInstanceUID'] + extracted_cols].copy()\nauto_labels.columns = ['StudyInstanceUID'] + label_cols\n\n# only keep studies where we got at least one real label extracted\nauto_labels = auto_labels.dropna(subset=label_cols, how='all')\nprint(f'Auto-labeled studies (at least 1 condition found): {len(auto_labels)}')\n\n# for now, fill any remaining missing conditions with 0 (assume \"not mentioned\" = not present)\n# this is an approximation, worth remembering later\nauto_labels[label_cols] = auto_labels[label_cols].fillna(0)\n\n# combine with your original 58 gold-labeled studies\ncombined = pd.concat([labeled[['StudyInstanceUID'] + label_cols], auto_labels], ignore_index=True)\ncombined = combined.drop_duplicates(subset='StudyInstanceUID')\nprint(f'Total combined training studies: {len(combined)}')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_ids = combined['StudyInstanceUID'].tolist()\n\nsag_combined = series[\n    (series['Anatomical_Plane'] == 'Sagittal') &\n    (series['StudyInstanceUID'].isin(combined_ids))\n]\none_per_study_combined = sag_combined.drop_duplicates('StudyInstanceUID')\nprint(f'Studies with a Sagittal series available: {len(one_per_study_combined)} / {len(combined)}')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\n\nstart = time.time()\nconverted = 0\nfailed = []\n\nfor _, row in one_per_study_combined.iterrows():\n    study_id = row['StudyInstanceUID']\n    series_id = row['SeriesInstanceUID']\n    out_path = f'{out_dir}/{study_id}.png'\n    \n    if os.path.exists(out_path):\n        converted += 1\n        continue  # skip if already converted (e.g. from our original 58)\n    \n    series_dir = f'/kaggle/input/competitions/rsna-knee-abnormality-detection/train_series/{study_id}/{series_id}'\n    try:\n        files = sorted(os.listdir(series_dir))\n        mid_file = files[len(files)//2]\n        dcm = pydicom.dcmread(f'{series_dir}/{mid_file}')\n        arr = dcm.pixel_array.astype(np.float32)\n        arr = (arr - arr.min()) / (arr.max() - arr.min() + 1e-6) * 255\n        Image.fromarray(arr.astype(np.uint8)).save(out_path)\n        converted += 1\n    except Exception as e:\n        failed.append((study_id, str(e)))\n\nprint(f'Converted: {converted}, Failed: {len(failed)}, Time: {time.time()-start:.1f}s')\nif failed:\n    print(failed[:5])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.vision.all import *\n\ncombined['fname'] = combined['StudyInstanceUID'] + '.png'\n\ndls = ImageDataLoaders.from_df(\n    combined,\n    path=out_dir,\n    fn_col='fname',\n    label_col=label_cols,\n    y_block=MultiCategoryBlock(encoded=True, vocab=label_cols),\n    item_tfms=Resize(224),\n    valid_pct=0.2,\n    bs=32,\n    seed=42\n)\ndls.show_batch()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learn = cnn_learner(dls, resnet34, pretrained=False, metrics=RocAucMulti())\nlearn.fit_one_cycle(15)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# per-label ROC AUC, not just averaged\ninterp = learn.get_preds(dl=dls.valid)\npreds, targs = interp\nfrom sklearn.metrics import roc_auc_score\n\nfor i, label in enumerate(label_cols):\n    try:\n        score = roc_auc_score(targs[:, i], preds[:, i])\n        print(f'{label}: {score:.3f}  (positives: {int(targs[:,i].sum())})')\n    except ValueError:\n        print(f'{label}: only one class present in validation set')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/competitions/rsna-knee-abnormality-detection/test.csv')\ntest_series = pd.read_csv('/kaggle/input/competitions/rsna-knee-abnormality-detection/test_series.csv')\nprint(test.shape)\ntest.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_sag = test_series[test_series['Anatomical_Plane'] == 'Sagittal']\ntest_one_per_study = test_sag.drop_duplicates('StudyInstanceUID')\nprint(len(test_one_per_study), '/', len(test))\ntest_one_per_study.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_out_dir = '/kaggle/working/test_pngs'\nos.makedirs(test_out_dir, exist_ok=True)\n\ndef convert_study(study_id, out_path):\n    study_series = test_series[test_series['StudyInstanceUID'] == study_id]\n    # prefer Sagittal, fall back to Coronal, then Axial, then whatever's available\n    for plane in ['Sagittal', 'Coronal', 'Axial']:\n        matches = study_series[study_series['Anatomical_Plane'] == plane]\n        if len(matches) > 0:\n            series_id = matches.iloc[0]['SeriesInstanceUID']\n            break\n    else:\n        series_id = study_series.iloc[0]['SeriesInstanceUID']  # last resort: just take whatever exists\n\n    series_dir = f'/kaggle/input/competitions/rsna-knee-abnormality-detection/test_series/{study_id}/{series_id}'\n    files = sorted(os.listdir(series_dir))\n    mid_file = files[len(files)//2]\n    dcm = pydicom.dcmread(f'{series_dir}/{mid_file}')\n    arr = dcm.pixel_array.astype(np.float32)\n    arr = (arr - arr.min()) / (arr.max() - arr.min() + 1e-6) * 255\n    Image.fromarray(arr.astype(np.uint8)).save(out_path)\n\nfailed_test = []\nfor study_id in test['StudyInstanceUID']:\n    out_path = f'{test_out_dir}/{study_id}.png'\n    try:\n        convert_study(study_id, out_path)\n    except Exception as e:\n        failed_test.append((study_id, str(e)))\n\nprint(f'Converted: {len(test) - len(failed_test)}, Failed: {len(failed_test)}')\nif failed_test:\n    print(failed_test)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_files = [f'{test_out_dir}/{sid}.png' for sid in test['StudyInstanceUID']]\ntest_dl = learn.dls.test_dl(test_files)\npreds, _ = learn.get_preds(dl=test_dl)\nprint(preds.shape)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = pd.DataFrame(preds.numpy(), columns=label_cols)\nsub.insert(0, 'StudyInstanceUID', test['StudyInstanceUID'].values)\nsub.to_csv('submission.csv', index=False)\nsub.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}