{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1 style=\"color: #6cb4e4;  text-align: center;  padding: 0.25em;  border-top: solid 2.5px #6cb4e4;  border-bottom: solid 2.5px #6cb4e4;  background: -webkit-repeating-linear-gradient(-45deg, #f0f8ff, #f0f8ff 3px,#e9f4ff 3px, #e9f4ff 7px);  background: repeating-linear-gradient(-45deg, #f0f8ff, #f0f8ff 3px,#e9f4ff 3px, #e9f4ff 7px);height:45px;\">\n<b>\nInf\n</b></h1> ","metadata":{"papermill":{"duration":0.005122,"end_time":"2022-08-29T06:34:27.903265","exception":false,"start_time":"2022-08-29T06:34:27.898143","status":"completed"},"pycharm":{"name":"#%% md\n"},"tags":[]}},{"cell_type":"markdown","source":"<a id=#cbb></a>\n<h2 style=\"color: #6cb4e4; background: #dfefff;  box-shadow: 0px 0px 0px 5px #dfefff;  border: dashed 4px white;  padding: 0.2em 0.5em;\">\n<b>\nModel1\n</b></h2> ","metadata":{}},{"cell_type":"code","source":"try:\n    import pylibjpeg\nexcept:\n    # Offline dependencies:\n    !mkdir -p /root/.cache/torch/hub/checkpoints/\n    !cp ../input/rsna-2022-whl/efficientnet_v2_s-dd5fe13b.pth  /root/.cache/torch/hub/checkpoints/\n\n    !pip install /kaggle/input/rsna-2022-whl/{pydicom-2.3.0-py3-none-any.whl,pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}\n    !pip install /kaggle/input/rsna-2022-whl/{torch-1.12.1-cp37-cp37m-manylinux1_x86_64.whl,torchvision-0.13.1-cp37-cp37m-manylinux1_x86_64.whl}","metadata":{"papermill":{"duration":110.198147,"end_time":"2022-08-29T06:36:18.107292","exception":false,"start_time":"2022-08-29T06:34:27.909145","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:05:02.013885Z","iopub.execute_input":"2022-10-27T13:05:02.014492Z","iopub.status.idle":"2022-10-27T13:06:49.293754Z","shell.execute_reply.started":"2022-10-27T13:05:02.014364Z","shell.execute_reply":"2022-10-27T13:06:49.292555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport glob\nimport os\nimport re\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport pydicom as dicom\nimport torch\nimport torchvision as tv\nfrom sklearn.model_selection import GroupKFold\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torchvision.models.feature_extraction import create_feature_extractor\nfrom tqdm.notebook import tqdm\n\nimport wandb\n\npd.set_option('display.max_rows', 1000)\npd.set_option('display.max_columns', 1000)\nplt.rcParams['figure.figsize'] = (20, 5)\n\n\n# Effnet\nWEIGHTS = tv.models.efficientnet.EfficientNet_V2_S_Weights.DEFAULT\nRSNA_2022_PATH = '../input/rsna-2022-cervical-spine-fracture-detection'\nTRAIN_IMAGES_PATH = f'{RSNA_2022_PATH}/train_images'\nTEST_IMAGES_PATH = f'{RSNA_2022_PATH}/test_images'\nEFFNET_CHECKPOINTS_PATH = '../input/rsna-train-a-510-20221015154535'\n\n# MODEL_NAMES = [f'effnetv2']\n\n# This notebook supports ensembles and single model predictions. Uncomment to switch to ensemble prediction:\nMODEL_NAMES = [f'effnetv2-f{i}' for i in range(5)]\n\n# Common\nFRAC_COLS = [f'C{i}_effnet_frac' for i in range(1, 8)]\nVERT_COLS = [f'C{i}_effnet_vert' for i in range(1, 8)]\n\ntry:\n    from kaggle_secrets import UserSecretsClient\n    IS_KAGGLE = True\nexcept:\n    IS_KAGGLE = False\n\n\n# Switch to offline for submission\nos.environ[\"WANDB_MODE\"] = \"offline\"\n\nif os.environ[\"WANDB_MODE\"] == \"online\":\n    if IS_KAGGLE:\n        os.environ['WANDB_API_KEY'] = UserSecretsClient().get_secret(\"WANDB_API_KEY\")\n\nif not IS_KAGGLE:\n    print('Running locally')\n    RSNA_2022_PATH = '/mnt/rsna2022'\n    TRAIN_IMAGES_PATH = '/mnt/rsna2022/train_images'\n    TEST_IMAGES_PATH = '/mnt/rsna2022/test_images'\n    METADATA_PATH = '/home/vslaykovsky/Downloads/'\n    EFFNET_CHECKPOINTS_PATH = 'frac_checkpoints'\n    os.environ['WANDB_API_KEY'] = 'yourkeyhere'\n\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nif DEVICE == 'cuda':\n    BATCH_SIZE = 512\nelse:\n    BATCH_SIZE = 512","metadata":{"papermill":{"duration":2.969425,"end_time":"2022-08-29T06:36:21.083846","exception":false,"start_time":"2022-08-29T06:36:18.114421","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:06:49.296191Z","iopub.execute_input":"2022-10-27T13:06:49.296578Z","iopub.status.idle":"2022-10-27T13:06:51.580792Z","shell.execute_reply.started":"2022-10-27T13:06:49.296539Z","shell.execute_reply":"2022-10-27T13:06:51.579626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision.transforms._presets import ImageClassification, InterpolationMode\nfrom functools import partial\n\nIMAGE_SIZE_TF = ImageClassification(\n    crop_size=384,\n    resize_size=384,\n    mean=[0.485, 0.456, 0.406],\n    std=[0.229, 0.224, 0.225],\n    interpolation=InterpolationMode.BILINEAR,\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:06:51.582891Z","iopub.execute_input":"2022-10-27T13:06:51.583804Z","iopub.status.idle":"2022-10-27T13:06:51.592232Z","shell.execute_reply.started":"2022-10-27T13:06:51.583762Z","shell.execute_reply":"2022-10-27T13:06:51.590043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_df_test():\n    df_test = pd.read_csv(f'{RSNA_2022_PATH}/test.csv')\n\n    if df_test.iloc[0].row_id == '1.2.826.0.1.3680043.10197_C1':\n        # test_images and test.csv are inconsistent in the dev dataset, fixing labels for the dev run.\n        df_test = pd.DataFrame({\n            \"row_id\": ['1.2.826.0.1.3680043.22327_C1', '1.2.826.0.1.3680043.25399_C1', '1.2.826.0.1.3680043.5876_C1'],\n            \"StudyInstanceUID\": ['1.2.826.0.1.3680043.22327', '1.2.826.0.1.3680043.25399', '1.2.826.0.1.3680043.5876'],\n            \"prediction_type\": [\"C1\", \"C1\", \"patient_overall\"]}\n        )\n    return df_test\n\ndf_test = load_df_test()\ndf_test","metadata":{"papermill":{"duration":0.045491,"end_time":"2022-08-29T06:36:21.148473","exception":false,"start_time":"2022-08-29T06:36:21.102982","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:06:51.594934Z","iopub.execute_input":"2022-10-27T13:06:51.595312Z","iopub.status.idle":"2022-10-27T13:06:51.649255Z","shell.execute_reply.started":"2022-10-27T13:06:51.595278Z","shell.execute_reply":"2022-10-27T13:06:51.644449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_slices = glob.glob(f'{TEST_IMAGES_PATH}/*/*')\ntest_slices = [re.findall(f'{TEST_IMAGES_PATH}/(.*)/(.*).dcm', s)[0] for s in test_slices]\ndf_test_slices = pd.DataFrame(data=test_slices, columns=['StudyInstanceUID', 'Slice']).astype({'Slice': int}).sort_values(['StudyInstanceUID', 'Slice']).reset_index(drop=True)\ndf_test_slices","metadata":{"papermill":{"duration":0.145663,"end_time":"2022-08-29T06:36:21.302098","exception":false,"start_time":"2022-08-29T06:36:21.156435","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:06:51.65133Z","iopub.execute_input":"2022-10-27T13:06:51.65437Z","iopub.status.idle":"2022-10-27T13:06:51.817217Z","shell.execute_reply.started":"2022-10-27T13:06:51.65433Z","shell.execute_reply":"2022-10-27T13:06:51.816105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path):\n    \"\"\"\n    This supports loading both regular and compressed JPEG images. \n    See the first sell with `pip install` commands for the necessary dependencies\n    \"\"\"\n    img=dicom.dcmread(path)\n    img.PhotometricInterpretation = 'YBR_FULL'\n    data = img.pixel_array    \n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data=(data * 255).astype(np.uint8)\n    return cv2.cvtColor(data, cv2.COLOR_GRAY2RGB), img\n\n\nim, meta = load_dicom(f'{TRAIN_IMAGES_PATH}/1.2.826.0.1.3680043.10001/1.dcm')\nplt.figure()\nplt.imshow(im)\nplt.title('regular image')\n\nim, meta = load_dicom(f'{TRAIN_IMAGES_PATH}/1.2.826.0.1.3680043.10014/1.dcm')\nplt.figure()\nplt.imshow(im)\nplt.title('jpeg')","metadata":{"papermill":{"duration":0.619183,"end_time":"2022-08-29T06:36:21.941518","exception":false,"start_time":"2022-08-29T06:36:21.322335","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:06:51.818639Z","iopub.execute_input":"2022-10-27T13:06:51.819186Z","iopub.status.idle":"2022-10-27T13:06:52.47295Z","shell.execute_reply.started":"2022-10-27T13:06:51.819155Z","shell.execute_reply":"2022-10-27T13:06:52.472035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EffnetDataSet(torch.utils.data.Dataset):    \n    def __init__(self, df, path, transforms=IMAGE_SIZE_TF):\n        super().__init__()\n        self.df = df\n        self.path = path\n        self.transforms = IMAGE_SIZE_TF\n        \n    def __getitem__(self, i):\n        path = os.path.join(self.path, self.df.iloc[i].StudyInstanceUID, f'{self.df.iloc[i].Slice}.dcm')        \n        \n        try:\n            img = load_dicom(path)[0]         \n            img = np.transpose(img, (2, 0, 1))  # Pytorch uses (batch, channel, height, width) order. Converting (height, width, channel) -> (channel, height, width)\n            if self.transforms is not None:\n                img = self.transforms(torch.as_tensor(img))\n        except Exception as ex:\n            print(ex)\n            return None\n        \n        if 'C1_fracture' in self.df:\n            frac_targets = torch.as_tensor(self.df.iloc[i][['C1_fracture', 'C2_fracture', 'C3_fracture', 'C4_fracture', 'C5_fracture', 'C6_fracture', 'C7_fracture']].astype('float32').values)\n            vert_targets = torch.as_tensor(self.df.iloc[i][['C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7']].astype('float32').values)\n            frac_targets = frac_targets * vert_targets   # we only enable targets that are visible on the current slice\n            return img, frac_targets, vert_targets\n        return img        \n    \n    def __len__(self):\n        return len(self.df)\n    ","metadata":{"papermill":{"duration":0.023057,"end_time":"2022-08-29T06:36:21.974596","exception":false,"start_time":"2022-08-29T06:36:21.951539","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:06:52.474541Z","iopub.execute_input":"2022-10-27T13:06:52.475173Z","iopub.status.idle":"2022-10-27T13:06:52.486785Z","shell.execute_reply.started":"2022-10-27T13:06:52.475135Z","shell.execute_reply":"2022-10-27T13:06:52.485553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only X values returned by the test dataset\nds_test = EffnetDataSet(df_test_slices, TEST_IMAGES_PATH, WEIGHTS.transforms())\nX = ds_test[42]\nX.shape","metadata":{"papermill":{"duration":0.04504,"end_time":"2022-08-29T06:36:22.02892","exception":false,"start_time":"2022-08-29T06:36:21.98388","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:06:52.489234Z","iopub.execute_input":"2022-10-27T13:06:52.48997Z","iopub.status.idle":"2022-10-27T13:06:52.534098Z","shell.execute_reply.started":"2022-10-27T13:06:52.489934Z","shell.execute_reply":"2022-10-27T13:06:52.533173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EffnetModel(torch.nn.Module):\n    def __init__(self):\n        super().__init__()\n        effnet = tv.models.efficientnet_v2_s()\n        self.model = create_feature_extractor(effnet, ['flatten'])\n        self.nn_fracture = torch.nn.Sequential(\n            torch.nn.Linear(1280, 7),\n        )\n        self.nn_vertebrae = torch.nn.Sequential(\n            torch.nn.Linear(1280, 7),\n        )\n\n    def forward(self, x):\n        # returns logits\n        x = self.model(x)['flatten']\n        return self.nn_fracture(x), self.nn_vertebrae(x)\n\n    def predict(self, x):\n        frac, vert = self.forward(x)\n        return torch.sigmoid(frac), torch.sigmoid(vert)\n\nmodel = EffnetModel()\nmodel.predict(torch.randn(1, 3, 384, 384))\ndel model","metadata":{"papermill":{"duration":2.094724,"end_time":"2022-08-29T06:36:24.151144","exception":false,"start_time":"2022-08-29T06:36:22.05642","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:06:52.535333Z","iopub.execute_input":"2022-10-27T13:06:52.535662Z","iopub.status.idle":"2022-10-27T13:06:53.820856Z","shell.execute_reply.started":"2022-10-27T13:06:52.535627Z","shell.execute_reply":"2022-10-27T13:06:53.819862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(model, name, path='.'):\n    data = torch.load(os.path.join(path, f'{name}.tph'), map_location=DEVICE)\n    model.load_state_dict(data)\n    return model","metadata":{"papermill":{"duration":0.019322,"end_time":"2022-08-29T06:36:24.180428","exception":false,"start_time":"2022-08-29T06:36:24.161106","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:06:53.825604Z","iopub.execute_input":"2022-10-27T13:06:53.825922Z","iopub.status.idle":"2022-10-27T13:06:53.831217Z","shell.execute_reply.started":"2022-10-27T13:06:53.825887Z","shell.execute_reply":"2022-10-27T13:06:53.830118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAMES","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:06:53.832882Z","iopub.execute_input":"2022-10-27T13:06:53.833568Z","iopub.status.idle":"2022-10-27T13:06:53.84356Z","shell.execute_reply.started":"2022-10-27T13:06:53.833533Z","shell.execute_reply":"2022-10-27T13:06:53.842409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAMES = [\n    'effnetv2-f0', \n    'effnetv2-f1', \n    'effnetv2-f2', \n    'effnetv2-f3', \n#     'effnetv2-f4',\n]","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:06:53.845386Z","iopub.execute_input":"2022-10-27T13:06:53.845807Z","iopub.status.idle":"2022-10-27T13:06:53.853851Z","shell.execute_reply.started":"2022-10-27T13:06:53.845766Z","shell.execute_reply":"2022-10-27T13:06:53.852806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"effnet_models = [load_model(EffnetModel(), name, EFFNET_CHECKPOINTS_PATH).to(DEVICE) for name in MODEL_NAMES]","metadata":{"papermill":{"duration":4.116502,"end_time":"2022-08-29T06:36:28.306547","exception":false,"start_time":"2022-08-29T06:36:24.190045","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:06:53.855356Z","iopub.execute_input":"2022-10-27T13:06:53.855738Z","iopub.status.idle":"2022-10-27T13:07:04.627867Z","shell.execute_reply.started":"2022-10-27T13:06:53.855704Z","shell.execute_reply":"2022-10-27T13:07:04.626861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import List\n\n\ndef predict_effnet(models: List[EffnetModel], ds, max_batches=1e9):\n    dl_test = torch.utils.data.DataLoader(ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=os.cpu_count())\n    for m in models:\n        m.eval()\n\n    with torch.no_grad():\n        predictions = []\n        for idx, X in enumerate(tqdm(dl_test, miniters=10)):\n            pred = torch.zeros(len(X), 14).to(DEVICE)\n            for m in models:\n                y1, y2 = m.predict(X.to(DEVICE))\n                pred += torch.concat([y1, y2], dim=1) / len(models)\n            predictions.append(pred)\n            if idx >= max_batches:\n                break\n        return torch.concat(predictions).cpu().numpy()\n\n# Quick test\npredict_effnet([EffnetModel().to(DEVICE)], ds_test, max_batches=2).shape","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":3.84226,"end_time":"2022-08-29T06:36:32.176344","exception":false,"start_time":"2022-08-29T06:36:28.334084","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:07:04.62945Z","iopub.execute_input":"2022-10-27T13:07:04.629805Z","iopub.status.idle":"2022-10-27T13:07:34.841197Z","shell.execute_reply.started":"2022-10-27T13:07:04.629769Z","shell.execute_reply":"2022-10-27T13:07:34.840007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"effnet_pred = predict_effnet(effnet_models, ds_test)\n\ndf_effnet_pred = pd.DataFrame(\n    data=effnet_pred, columns=[f'C{i}_effnet_frac' for i in range(1, 8)] + [f'C{i}_effnet_vert' for i in range(1, 8)]\n)","metadata":{"papermill":{"duration":23.579326,"end_time":"2022-08-29T06:36:55.765273","exception":false,"start_time":"2022-08-29T06:36:32.185947","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:07:34.842923Z","iopub.execute_input":"2022-10-27T13:07:34.843668Z","iopub.status.idle":"2022-10-27T13:08:17.028421Z","shell.execute_reply.started":"2022-10-27T13:07:34.843627Z","shell.execute_reply":"2022-10-27T13:08:17.026669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_pred = pd.concat([df_test_slices, df_effnet_pred], axis=1).sort_values(['StudyInstanceUID', 'Slice'])\ndf_test_pred","metadata":{"papermill":{"duration":0.063127,"end_time":"2022-08-29T06:36:55.844489","exception":false,"start_time":"2022-08-29T06:36:55.781362","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:08:17.030515Z","iopub.execute_input":"2022-10-27T13:08:17.030924Z","iopub.status.idle":"2022-10-27T13:08:17.064197Z","shell.execute_reply.started":"2022-10-27T13:08:17.030882Z","shell.execute_reply":"2022-10-27T13:08:17.063361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_sample_patient(df_pred):\n    patient = np.random.choice(df_pred.StudyInstanceUID)\n    df = df_pred.query('StudyInstanceUID == @patient').reset_index()\n\n    df[[f'C{i}_effnet_frac' for i in range(1, 8)]].plot(\n        title=f'Patient {patient}, fracture prediction',\n        ax=(plt.subplot(1, 2, 1)))\n\n    df[[f'C{i}_effnet_vert' for i in range(1, 8)]].plot(\n        title=f'Patient {patient}, vertebrae prediction',\n        ax=plt.subplot(1, 2, 2)\n    )\n\nplot_sample_patient(df_test_pred)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.629707,"end_time":"2022-08-29T06:36:56.489965","exception":false,"start_time":"2022-08-29T06:36:55.860258","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:08:17.065811Z","iopub.execute_input":"2022-10-27T13:08:17.066202Z","iopub.status.idle":"2022-10-27T13:08:17.569902Z","shell.execute_reply.started":"2022-10-27T13:08:17.066166Z","shell.execute_reply":"2022-10-27T13:08:17.568947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef patient_prediction(df):\n    c1c7 = np.average(df[FRAC_COLS].values, axis=0, weights=df[VERT_COLS].values)\n    pred_patient_overall = 1 - np.prod(1 - c1c7)\n    return pd.Series(data=np.concatenate([[pred_patient_overall], c1c7]), index=['patient_overall'] + [f'C{i}' for i in range(1, 8)])\n\ndf_patient_pred = df_test_pred.groupby('StudyInstanceUID').apply(lambda df: patient_prediction(df))\ndf_patient_pred","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.038032,"end_time":"2022-08-29T06:36:56.54117","exception":false,"start_time":"2022-08-29T06:36:56.503138","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:08:17.571484Z","iopub.execute_input":"2022-10-27T13:08:17.572194Z","iopub.status.idle":"2022-10-27T13:08:17.599641Z","shell.execute_reply.started":"2022-10-27T13:08:17.572155Z","shell.execute_reply":"2022-10-27T13:08:17.598493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = df_test.copy()\ndf_sub = df_sub.set_index('StudyInstanceUID').join(df_patient_pred)\ndf_sub['fractured'] = df_sub.apply(lambda r: r[r.prediction_type], axis=1)\ndf_sub","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.036645,"end_time":"2022-08-29T06:36:56.590835","exception":false,"start_time":"2022-08-29T06:36:56.55419","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:08:17.60138Z","iopub.execute_input":"2022-10-27T13:08:17.602138Z","iopub.status.idle":"2022-10-27T13:08:17.62744Z","shell.execute_reply.started":"2022-10-27T13:08:17.602099Z","shell.execute_reply":"2022-10-27T13:08:17.62658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_sub[['row_id', 'fractured']].to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":0.024447,"end_time":"2022-08-29T06:36:56.629233","exception":false,"start_time":"2022-08-29T06:36:56.604786","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-10-27T13:08:17.628885Z","iopub.execute_input":"2022-10-27T13:08:17.629276Z","iopub.status.idle":"2022-10-27T13:08:17.634936Z","shell.execute_reply.started":"2022-10-27T13:08:17.629223Z","shell.execute_reply":"2022-10-27T13:08:17.633979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=#cbb></a>\n<h2 style=\"color: #6cb4e4; background: #dfefff;  box-shadow: 0px 0px 0px 5px #dfefff;  border: dashed 4px white;  padding: 0.2em 0.5em;\">\n<b>\nModel2\n</b></h2> ","metadata":{}},{"cell_type":"code","source":"import gc\nimport glob\nimport os\nimport re\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport pydicom as dicom\nimport torch\nimport torchvision as tv\nfrom sklearn.model_selection import GroupKFold\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torchvision.models.feature_extraction import create_feature_extractor\nfrom tqdm.notebook import tqdm\n\nimport wandb\n\npd.set_option('display.max_rows', 1000)\npd.set_option('display.max_columns', 1000)\nplt.rcParams['figure.figsize'] = (20, 5)\n\n\n# Effnet\nWEIGHTS = tv.models.swin_transformer.Swin_S_Weights.DEFAULT\nRSNA_2022_PATH = '../input/rsna-2022-cervical-spine-fracture-detection'\nTRAIN_IMAGES_PATH = f'{RSNA_2022_PATH}/train_images'\nTEST_IMAGES_PATH = f'{RSNA_2022_PATH}/test_images'\nEFFNET_CHECKPOINTS_PATH = '../input/rsna-train-e-101-20221027125156'\n\n# MODEL_NAMES = [f'effnetv2']\n\n# This notebook supports ensembles and single model predictions. Uncomment to switch to ensemble prediction:\nMODEL_NAMES = [f'effnetv2-f{i}' for i in range(5)]\n\n# Common\nFRAC_COLS = [f'C{i}_effnet_frac' for i in range(1, 8)]\nVERT_COLS = [f'C{i}_effnet_vert' for i in range(1, 8)]\n\ntry:\n    from kaggle_secrets import UserSecretsClient\n    IS_KAGGLE = True\nexcept:\n    IS_KAGGLE = False\n\n\n# Switch to offline for submission\nos.environ[\"WANDB_MODE\"] = \"offline\"\n\nif os.environ[\"WANDB_MODE\"] == \"online\":\n    if IS_KAGGLE:\n        os.environ['WANDB_API_KEY'] = UserSecretsClient().get_secret(\"WANDB_API_KEY\")\n\nif not IS_KAGGLE:\n    print('Running locally')\n    RSNA_2022_PATH = '/mnt/rsna2022'\n    TRAIN_IMAGES_PATH = '/mnt/rsna2022/train_images'\n    TEST_IMAGES_PATH = '/mnt/rsna2022/test_images'\n    METADATA_PATH = '/home/vslaykovsky/Downloads/'\n    EFFNET_CHECKPOINTS_PATH = 'frac_checkpoints'\n    os.environ['WANDB_API_KEY'] = 'yourkeyhere'\n\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nif DEVICE == 'cuda':\n    BATCH_SIZE = 256\nelse:\n    BATCH_SIZE = 256","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:17.636928Z","iopub.execute_input":"2022-10-27T13:08:17.637209Z","iopub.status.idle":"2022-10-27T13:08:17.651124Z","shell.execute_reply.started":"2022-10-27T13:08:17.637184Z","shell.execute_reply":"2022-10-27T13:08:17.650042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_df_test():\n    df_test = pd.read_csv(f'{RSNA_2022_PATH}/test.csv')\n\n    if df_test.iloc[0].row_id == '1.2.826.0.1.3680043.10197_C1':\n        # test_images and test.csv are inconsistent in the dev dataset, fixing labels for the dev run.\n        df_test = pd.DataFrame({\n            \"row_id\": ['1.2.826.0.1.3680043.22327_C1', '1.2.826.0.1.3680043.25399_C1', '1.2.826.0.1.3680043.5876_C1'],\n            \"StudyInstanceUID\": ['1.2.826.0.1.3680043.22327', '1.2.826.0.1.3680043.25399', '1.2.826.0.1.3680043.5876'],\n            \"prediction_type\": [\"C1\", \"C1\", \"patient_overall\"]}\n        )\n    return df_test\n\ndf_test = load_df_test()\ndf_test","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:17.652859Z","iopub.execute_input":"2022-10-27T13:08:17.653568Z","iopub.status.idle":"2022-10-27T13:08:17.673988Z","shell.execute_reply.started":"2022-10-27T13:08:17.653534Z","shell.execute_reply":"2022-10-27T13:08:17.673013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_slices = glob.glob(f'{TEST_IMAGES_PATH}/*/*')\ntest_slices = [re.findall(f'{TEST_IMAGES_PATH}/(.*)/(.*).dcm', s)[0] for s in test_slices]\ndf_test_slices = pd.DataFrame(data=test_slices, columns=['StudyInstanceUID', 'Slice']).astype({'Slice': int}).sort_values(['StudyInstanceUID', 'Slice']).reset_index(drop=True)\ndf_test_slices","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:17.675516Z","iopub.execute_input":"2022-10-27T13:08:17.675787Z","iopub.status.idle":"2022-10-27T13:08:17.703973Z","shell.execute_reply.started":"2022-10-27T13:08:17.675763Z","shell.execute_reply":"2022-10-27T13:08:17.703098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path):\n    \"\"\"\n    This supports loading both regular and compressed JPEG images. \n    See the first sell with `pip install` commands for the necessary dependencies\n    \"\"\"\n    img=dicom.dcmread(path)\n    img.PhotometricInterpretation = 'YBR_FULL'\n    data = img.pixel_array    \n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data=(data * 255).astype(np.uint8)\n    return cv2.cvtColor(data, cv2.COLOR_GRAY2RGB), img\n\n\nim, meta = load_dicom(f'{TRAIN_IMAGES_PATH}/1.2.826.0.1.3680043.10001/1.dcm')\nplt.figure()\nplt.imshow(im)\nplt.title('regular image')\n\nim, meta = load_dicom(f'{TRAIN_IMAGES_PATH}/1.2.826.0.1.3680043.10014/1.dcm')\nplt.figure()\nplt.imshow(im)\nplt.title('jpeg')","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:17.705651Z","iopub.execute_input":"2022-10-27T13:08:17.706071Z","iopub.status.idle":"2022-10-27T13:08:18.23509Z","shell.execute_reply.started":"2022-10-27T13:08:17.706018Z","shell.execute_reply":"2022-10-27T13:08:18.234181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EffnetDataSet(torch.utils.data.Dataset):    \n    def __init__(self, df, path, transforms=None):\n        super().__init__()\n        self.df = df\n        self.path = path\n        self.transforms = transforms\n        \n    def __getitem__(self, i):\n        path = os.path.join(self.path, self.df.iloc[i].StudyInstanceUID, f'{self.df.iloc[i].Slice}.dcm')        \n        \n        try:\n            img = load_dicom(path)[0]         \n            img = np.transpose(img, (2, 0, 1))  # Pytorch uses (batch, channel, height, width) order. Converting (height, width, channel) -> (channel, height, width)\n            if self.transforms is not None:\n                img = self.transforms(torch.as_tensor(img))\n        except Exception as ex:\n            print(ex)\n            return None\n        \n        if 'C1_fracture' in self.df:\n            frac_targets = torch.as_tensor(self.df.iloc[i][['C1_fracture', 'C2_fracture', 'C3_fracture', 'C4_fracture', 'C5_fracture', 'C6_fracture', 'C7_fracture']].astype('float32').values)\n            vert_targets = torch.as_tensor(self.df.iloc[i][['C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7']].astype('float32').values)\n            frac_targets = frac_targets * vert_targets   # we only enable targets that are visible on the current slice\n            return img, frac_targets, vert_targets\n        return img        \n    \n    def __len__(self):\n        return len(self.df)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:18.236693Z","iopub.execute_input":"2022-10-27T13:08:18.23731Z","iopub.status.idle":"2022-10-27T13:08:18.247869Z","shell.execute_reply.started":"2022-10-27T13:08:18.23727Z","shell.execute_reply":"2022-10-27T13:08:18.246868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only X values returned by the test dataset\nds_test = EffnetDataSet(df_test_slices, TEST_IMAGES_PATH, WEIGHTS.transforms())\nX = ds_test[42]\nX.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:18.249387Z","iopub.execute_input":"2022-10-27T13:08:18.249829Z","iopub.status.idle":"2022-10-27T13:08:18.278511Z","shell.execute_reply.started":"2022-10-27T13:08:18.249793Z","shell.execute_reply":"2022-10-27T13:08:18.277652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EffnetModel(torch.nn.Module):\n    def __init__(self):\n        super().__init__()\n        effnet = tv.models.swin_s()\n        self.model = create_feature_extractor(effnet, ['flatten'])\n        self.nn_fracture = torch.nn.Sequential(\n            torch.nn.Linear(768, 7),\n        )\n        self.nn_vertebrae = torch.nn.Sequential(\n            torch.nn.Linear(768, 7),\n        )\n\n    def forward(self, x):\n        # returns logits\n        x = self.model(x)['flatten']\n        return self.nn_fracture(x), self.nn_vertebrae(x)\n\n    def predict(self, x):\n        frac, vert = self.forward(x)\n        return torch.sigmoid(frac), torch.sigmoid(vert)\n\nmodel = EffnetModel()\nmodel.predict(torch.randn(1, 3, 384, 384))\ndel model","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:18.280463Z","iopub.execute_input":"2022-10-27T13:08:18.2808Z","iopub.status.idle":"2022-10-27T13:08:22.398686Z","shell.execute_reply.started":"2022-10-27T13:08:18.280767Z","shell.execute_reply":"2022-10-27T13:08:22.397678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(model, name, path='.'):\n    data = torch.load(os.path.join(path, f'{name}.tph'), map_location=DEVICE)\n    model.load_state_dict(data)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:22.404534Z","iopub.execute_input":"2022-10-27T13:08:22.40483Z","iopub.status.idle":"2022-10-27T13:08:22.414145Z","shell.execute_reply.started":"2022-10-27T13:08:22.404803Z","shell.execute_reply":"2022-10-27T13:08:22.413158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAMES","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:22.415667Z","iopub.execute_input":"2022-10-27T13:08:22.416354Z","iopub.status.idle":"2022-10-27T13:08:22.427891Z","shell.execute_reply.started":"2022-10-27T13:08:22.41628Z","shell.execute_reply":"2022-10-27T13:08:22.426882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAMES = [\n    'effnetv2-f0', \n    'effnetv2-f1', \n    'effnetv2-f2', \n    'effnetv2-f3', \n#     'effnetv2-f4',\n]","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:22.429372Z","iopub.execute_input":"2022-10-27T13:08:22.429908Z","iopub.status.idle":"2022-10-27T13:08:22.436553Z","shell.execute_reply.started":"2022-10-27T13:08:22.42987Z","shell.execute_reply":"2022-10-27T13:08:22.43562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"effnet_models = [load_model(EffnetModel(), name, EFFNET_CHECKPOINTS_PATH).to(DEVICE) for name in MODEL_NAMES]","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:22.437824Z","iopub.execute_input":"2022-10-27T13:08:22.438722Z","iopub.status.idle":"2022-10-27T13:08:37.036619Z","shell.execute_reply.started":"2022-10-27T13:08:22.43868Z","shell.execute_reply":"2022-10-27T13:08:37.035565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import List\n\n\ndef predict_effnet(models: List[EffnetModel], ds, max_batches=1e9):\n    dl_test = torch.utils.data.DataLoader(ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=os.cpu_count())\n    for m in models:\n        m.eval()\n\n    with torch.no_grad():\n        predictions = []\n        for idx, X in enumerate(tqdm(dl_test, miniters=10)):\n            pred = torch.zeros(len(X), 14).to(DEVICE)\n            for m in models:\n                y1, y2 = m.predict(X.to(DEVICE))\n                pred += torch.concat([y1, y2], dim=1) / len(models)\n            predictions.append(pred)\n            if idx >= max_batches:\n                break\n        return torch.concat(predictions).cpu().numpy()\n\n# Quick test\npredict_effnet([EffnetModel().to(DEVICE)], ds_test, max_batches=2).shape","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:37.039415Z","iopub.execute_input":"2022-10-27T13:08:37.039887Z","iopub.status.idle":"2022-10-27T13:08:53.404233Z","shell.execute_reply.started":"2022-10-27T13:08:37.039841Z","shell.execute_reply":"2022-10-27T13:08:53.402955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"effnet_pred = predict_effnet(effnet_models, ds_test)\n\ndf_effnet_pred = pd.DataFrame(\n    data=effnet_pred, columns=[f'C{i}_effnet_frac' for i in range(1, 8)] + [f'C{i}_effnet_vert' for i in range(1, 8)]\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:08:53.406175Z","iopub.execute_input":"2022-10-27T13:08:53.407307Z","iopub.status.idle":"2022-10-27T13:09:21.905891Z","shell.execute_reply.started":"2022-10-27T13:08:53.407265Z","shell.execute_reply":"2022-10-27T13:09:21.904776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_pred = pd.concat([df_test_slices, df_effnet_pred], axis=1).sort_values(['StudyInstanceUID', 'Slice'])\ndf_test_pred","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:09:21.90769Z","iopub.execute_input":"2022-10-27T13:09:21.908298Z","iopub.status.idle":"2022-10-27T13:09:21.940164Z","shell.execute_reply.started":"2022-10-27T13:09:21.908265Z","shell.execute_reply":"2022-10-27T13:09:21.939058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_sample_patient(df_pred):\n    patient = np.random.choice(df_pred.StudyInstanceUID)\n    df = df_pred.query('StudyInstanceUID == @patient').reset_index()\n\n    df[[f'C{i}_effnet_frac' for i in range(1, 8)]].plot(\n        title=f'Patient {patient}, fracture prediction',\n        ax=(plt.subplot(1, 2, 1)))\n\n    df[[f'C{i}_effnet_vert' for i in range(1, 8)]].plot(\n        title=f'Patient {patient}, vertebrae prediction',\n        ax=plt.subplot(1, 2, 2)\n    )\n\nplot_sample_patient(df_test_pred)","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:09:21.941545Z","iopub.execute_input":"2022-10-27T13:09:21.942227Z","iopub.status.idle":"2022-10-27T13:09:22.469903Z","shell.execute_reply.started":"2022-10-27T13:09:21.942188Z","shell.execute_reply":"2022-10-27T13:09:22.46895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef patient_prediction(df):\n    c1c7 = np.average(df[FRAC_COLS].values, axis=0, weights=df[VERT_COLS].values)\n    pred_patient_overall = 1 - np.prod(1 - c1c7)\n    return pd.Series(data=np.concatenate([[pred_patient_overall], c1c7]), index=['patient_overall'] + [f'C{i}' for i in range(1, 8)])\n\ndf_patient_pred = df_test_pred.groupby('StudyInstanceUID').apply(lambda df: patient_prediction(df))\ndf_patient_pred","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:09:22.471527Z","iopub.execute_input":"2022-10-27T13:09:22.472242Z","iopub.status.idle":"2022-10-27T13:09:22.497597Z","shell.execute_reply.started":"2022-10-27T13:09:22.472203Z","shell.execute_reply":"2022-10-27T13:09:22.496529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = df_test.copy()\nsubmission = submission.set_index('StudyInstanceUID').join(df_patient_pred)\nsubmission['fractured'] = submission.apply(lambda r: r[r.prediction_type], axis=1)\nsubmission.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:09:22.499268Z","iopub.execute_input":"2022-10-27T13:09:22.499947Z","iopub.status.idle":"2022-10-27T13:09:22.522523Z","shell.execute_reply.started":"2022-10-27T13:09:22.499909Z","shell.execute_reply":"2022-10-27T13:09:22.520948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = submission[['row_id', 'fractured']].reset_index(drop=True)\n# submission","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:09:22.524243Z","iopub.execute_input":"2022-10-27T13:09:22.524633Z","iopub.status.idle":"2022-10-27T13:09:22.529096Z","shell.execute_reply.started":"2022-10-27T13:09:22.524596Z","shell.execute_reply":"2022-10-27T13:09:22.527944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=#cbb></a>\n<h2 style=\"color: #6cb4e4; background: #dfefff;  box-shadow: 0px 0px 0px 5px #dfefff;  border: dashed 4px white;  padding: 0.2em 0.5em;\">\n<b>\nBlend\n</b></h2> ","metadata":{}},{"cell_type":"code","source":"df_sub","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:09:22.530683Z","iopub.execute_input":"2022-10-27T13:09:22.531216Z","iopub.status.idle":"2022-10-27T13:09:22.550634Z","shell.execute_reply.started":"2022-10-27T13:09:22.531179Z","shell.execute_reply":"2022-10-27T13:09:22.550011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:09:22.551976Z","iopub.execute_input":"2022-10-27T13:09:22.552596Z","iopub.status.idle":"2022-10-27T13:09:22.567565Z","shell.execute_reply.started":"2022-10-27T13:09:22.552562Z","shell.execute_reply":"2022-10-27T13:09:22.566457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sub = df_sub[['row_id', 'fractured']].copy()\nsub = submission.copy()","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:09:22.569142Z","iopub.execute_input":"2022-10-27T13:09:22.569539Z","iopub.status.idle":"2022-10-27T13:09:22.576626Z","shell.execute_reply.started":"2022-10-27T13:09:22.569506Z","shell.execute_reply":"2022-10-27T13:09:22.575622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_W_1=0.95\nMODEL_W_2=0.05\n\nfor i in range(df_sub.shape[0]):\n    idx = df_sub['row_id'][i]\n    val1 = df_sub['fractured'][df_sub['row_id'] == str(idx)].values[0]\n    val2 = submission['fractured'][submission['row_id'] == str(idx)].values[0]\n\n    sub['fractured'][sub['row_id'] == str(idx)] = val1*MODEL_W_1 + val2*MODEL_W_2\n    \nsub.to_csv(\"submission.csv\", index=False)\nsub[['row_id', 'fractured']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-27T13:09:22.578346Z","iopub.execute_input":"2022-10-27T13:09:22.578796Z","iopub.status.idle":"2022-10-27T13:09:22.600561Z","shell.execute_reply.started":"2022-10-27T13:09:22.578759Z","shell.execute_reply":"2022-10-27T13:09:22.59955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}