{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# from apex import amp\nimport glob\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.optim as optim\nfrom albumentations import Compose, ShiftScaleRotate, Resize, HorizontalFlip, RandomBrightnessContrast, \\\n    Normalize\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.decomposition import PCA\nfrom sklearn.metrics import roc_auc_score\nfrom torch.utils.data import Dataset\nfrom tqdm import tqdm\nfrom datasets import IntracranialDataset\nfrom models import ResNeXtModel\n\nsaved_model_dir = '../input/resnext32x8dcheckpoint/'\n\ndir_csv = '../input/rsna-intracranial-hemorrhage-detection'\ntest_images_dir = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_test/'\ntrain_images_dir = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train'\ntrain_metadata_csv = '../input/rsna-intracranial-sequence-metadata/train_metadata_noidx.csv'\ntest_metadata_csv = '../input/rsna-intracranial-sequence-metadata/test_metadata_noidx.csv'\n\nn_classes = 6\nn_epochs = 3\nbatch_size = 32\n\nCOLS = ['epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural', 'any']\n\n# Read train and test data\ntrain = pd.read_csv(os.path.join(dir_csv, 'stage_2_train.csv'))\ntest = pd.read_csv(os.path.join(dir_csv, 'stage_2_sample_submission.csv'))\n\n# Read metadata for train/validation split\ntest_metadata_noidx = pd.read_csv(test_metadata_csv)\ntrain_metadata_noidx = pd.read_csv(train_metadata_csv)\n\n# Prepare train table\ntrain[['ID', 'Image', 'Diagnosis']] = train['ID'].str.split('_', expand=True)\ntrain = train[['Image', 'Diagnosis', 'Label']]\ntrain.drop_duplicates(inplace=True)\ntrain = train.pivot(index='Image', columns='Diagnosis', values='Label').reset_index()\ntrain['Image'] = 'ID_' + train['Image']\n\n# Remove invalid PNGs\npng = glob.glob(os.path.join(train_images_dir, '*.dcm'))\npng = [os.path.basename(png)[:-4] for png in png]\npng = np.array(png)\n\ntrain = train[train['Image'].isin(png)]\n\nmerged_train = pd.merge(left=train, right=train_metadata_noidx, how='left', left_on='Image', right_on='ImageId')\n\ntrain_series = train_metadata_noidx['SeriesInstanceUID'].unique()\nvalid_series = train_series[21000:]\ntrain_series = train_series[:21000]\n\nprint(len(train_series))\nprint(len(valid_series))\n\ntrain_df = merged_train[merged_train['SeriesInstanceUID'].isin(train_series)]\nvalid_df = merged_train[merged_train['SeriesInstanceUID'].isin(valid_series)]\n\nprint(len(train_df))\nprint(len(valid_df))\n\ntrain_df.to_csv('train.csv', index=False)\nprint(train_df['any'].value_counts())\nvalid_df.to_csv('valid.csv', index=False)\nprint(valid_df['any'].value_counts())\n\n# Prepare test table\ntest[['ID', 'Image', 'Diagnosis']] = test['ID'].str.split('_', expand=True)\ntest['Image'] = 'ID_' + test['Image']\ntest = test[['Image', 'Label']]\ntest.drop_duplicates(inplace=True)\n\ntest.to_csv('test.csv', index=False)\n\n# Data loaders\ntransform_train = Compose([Resize(256, 256),\n                           Normalize(mean=[0.1738, 0.1433, 0.1970], std=[0.3161, 0.2850, 0.3111], max_pixel_value=1.),\n                           HorizontalFlip(),\n                           ShiftScaleRotate(),\n                           RandomBrightnessContrast(),\n                           ToTensorV2()])\n\ntransform_test = Compose([Resize(256, 256),\n                          Normalize(mean=[0.1738, 0.1433, 0.1970], std=[0.3161, 0.2850, 0.3111], max_pixel_value=1.),\n                          ToTensorV2()])\n\ntransform_tta = Compose([Resize(256, 256),\n                         HorizontalFlip(),\n                         ShiftScaleRotate(),\n                         Normalize(mean=[0.1738, 0.1433, 0.1970], std=[0.3161, 0.2850, 0.3111], max_pixel_value=1.),\n                         ToTensorV2()])\n\ntrain_dataset = IntracranialDataset(\n    csv_file='train.csv', path=train_images_dir, transform=transform_train, labels=True)\n\nvalid_dataset = IntracranialDataset(\n    csv_file='valid.csv', path=train_images_dir, transform=transform_train, labels=True)\n\ntest_dataset = IntracranialDataset(\n    csv_file='test.csv', path=test_images_dir, transform=transform_test, labels=False)\n\ndata_loader_train = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\ndata_loader_valid = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\ndata_loader_test = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n\nprint(len(train_dataset))\nprint(len(valid_dataset))\nprint(len(test_dataset))\n\nprint(len(data_loader_train))\nprint(len(data_loader_valid))\nprint(len(data_loader_test))\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nmodel = ResNeXtModel()\n\nmodel.to(device)\n\ncriterion = torch.nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(model.parameters(), lr=1e-5)\n\n# model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\")\n\nfor epoch in range(n_epochs):\n\n    print('Epoch {}/{}'.format(epoch + 1, n_epochs))\n    print('-' * 10)\n\n    model.train()\n    tr_loss = 0\n\n    for step, batch in enumerate(data_loader_train):\n        inputs = batch[\"image\"]\n        labels = batch[\"labels\"]\n\n        inputs = inputs.to(device, dtype=torch.float)\n        labels = labels.to(device, dtype=torch.float)\n\n        outputs, _ = model(inputs)\n        loss = criterion(outputs, labels)\n\n        with amp.scale_loss(loss, optimizer) as scaled_loss:\n            scaled_loss.backward()\n\n        tr_loss += loss.item()\n\n        optimizer.step()\n        optimizer.zero_grad()\n\n        if step % 512 == 0:\n            epoch_loss = tr_loss / (step + 1)\n            print('Training Loss at {}: {:.4f}'.format(step, epoch_loss))\n\n    epoch_loss = tr_loss / len(data_loader_train)\n    print('Training Loss: {:.4f}'.format(epoch_loss))\n    print('-----------------------')\n\n    model.eval()\n    tr_loss = 0\n\n    auc_preds = []\n    auc_truths = []\n\n    for step, batch in enumerate(data_loader_valid):\n        inputs = batch[\"image\"]\n        labels = batch[\"labels\"]\n\n        inputs = inputs.to(device, dtype=torch.float)\n        labels = labels.to(device, dtype=torch.float)\n\n        outputs, _ = model(inputs)\n        loss = criterion(outputs, labels)\n\n        tr_loss += loss.item()\n\n        auc_preds.append(outputs.view(-1, 6).detach().cpu().numpy())\n        auc_truths.append(labels.view(-1, 6).detach().cpu().numpy())\n\n    epoch_loss = tr_loss / len(data_loader_valid)\n    print('Validation Loss: {:.4f}'.format(epoch_loss))\n\n    roc_preds = np.concatenate(auc_preds)\n\n    roc_truths = np.concatenate(auc_truths)\n\n    for tp in range(0, 6):\n        print(COLS[tp], roc_auc_score(roc_truths[:, tp], roc_preds[:, tp]), )\n    print('-----------------------')\n\n# Save checkpoint\ncheckpoint = {\n    'model': model.state_dict(),\n    'optimizer': optimizer.state_dict(),\n    # 'amp': amp.state_dict()\n}\ntorch.save(checkpoint, 'model.pt')\n\n# Save embeddings/predictions\nPCA_BATCHES = 1000\nmodel.eval()\n\ntrain_embed_dict = {}\n\nfor i, x_batch in enumerate(tqdm(data_loader_train)):\n    x_images = x_batch['image_id']\n    x_batch = x_batch[\"image\"]\n    x_batch = x_batch.to(device, dtype=torch.float)\n\n    if i > PCA_BATCHES:\n        break\n\n    with torch.no_grad():\n        _, embed = model(x_batch)\n\n        for x, y in zip(x_images, embed):\n            e = y.squeeze().detach().cpu().numpy()\n            train_embed_dict[x] = e\n\nemb_stat = np.array(list(train_embed_dict.values()))\nprint(np.mean(emb_stat), np.std(emb_stat))  # 0.4707987 0.7724904\n\n# pca = PCA()\n# pca.fit(emb_stat)\n# plt.figure()\n# plt.plot(np.cumsum(pca.explained_variance_ratio_))\n# plt.xlabel('Number of Components')\n# plt.ylabel('Variance (%)')\n# plt.title('Explained Variance')\n# plt.show()\n\npca = PCA(n_components=120)\n\npca.fit(emb_stat)\n\nmodel.eval()\n\n# TRAIN\ntrain_pred_dict = {}\ntrain_embed_dict = {}\n\nfor i, x_batch in enumerate(tqdm(data_loader_train)):\n    x_images = x_batch['image_id']\n    x_batch = x_batch[\"image\"]\n    x_batch = x_batch.to(device, dtype=torch.float)\n\n    with torch.no_grad():\n\n        pred, embed = model(x_batch)\n        pred = torch.sigmoid(pred)\n\n        for x, y in zip(x_images, pred):\n            train_pred_dict[x] = y.detach().cpu().numpy()\n        for x, y in zip(x_images, embed):\n            e = y.squeeze().detach().cpu().numpy()\n            e = np.expand_dims(e, axis=0)\n            train_embed_dict[x] = pca.transform(e)[0]\n\ntrain_embed_df = pd.DataFrame.from_dict(train_embed_dict, orient='index')\ntrain_embed_df.to_csv('train_embeds.csv')\n\ntrain_pred_df = pd.DataFrame.from_dict(train_pred_dict, orient='index')\ntrain_pred_df.to_csv('train_preds.csv')\n\n# VALID\nvalid_pred_dict = {}\nvalid_embed_dict = {}\n\nfor i, x_batch in enumerate(tqdm(data_loader_valid)):\n    x_images = x_batch['image_id']\n    x_batch = x_batch[\"image\"]\n    x_batch = x_batch.to(device, dtype=torch.float)\n\n    with torch.no_grad():\n\n        pred, embed = model(x_batch)\n        pred = torch.sigmoid(pred)\n\n        for x, y in zip(x_images, pred):\n            valid_pred_dict[x] = y.detach().cpu().numpy()\n        for x, y in zip(x_images, embed):\n            e = y.squeeze().detach().cpu().numpy()\n            e = np.expand_dims(e, axis=0)\n            valid_embed_dict[x] = pca.transform(e)[0]\n\nvalid_embed_df = pd.DataFrame.from_dict(valid_embed_dict, orient='index')\nvalid_embed_df.to_csv('valid_embeds.csv')\n\nvalid_pred_df = pd.DataFrame.from_dict(valid_pred_dict, orient='index')\nvalid_pred_df.to_csv('valid_preds.csv')\n\n# TEST\ntest_pred_dict = {}\ntest_embed_dict = {}\n\nfor i, x_batch in enumerate(tqdm(data_loader_valid)):\n    x_images = x_batch['image_id']\n    x_batch = x_batch[\"image\"]\n    x_batch = x_batch.to(device, dtype=torch.float)\n\n    with torch.no_grad():\n\n        pred, embed = model(x_batch)\n        pred = torch.sigmoid(pred)\n\n        for x, y in zip(x_images, pred):\n            test_pred_dict[x] = y.detach().cpu().numpy()\n        for x, y in zip(x_images, embed):\n            e = y.squeeze().detach().cpu().numpy()\n            e = np.expand_dims(e, axis=0)\n            test_embed_dict[x] = pca.transform(e)[0]\n\ntest_embed_df = pd.DataFrame.from_dict(test_embed_dict, orient='index')\ntest_embed_df.to_csv('test_embeds.csv')\n\ntest_pred_df = pd.DataFrame.from_dict(test_pred_dict, orient='index')\ntest_pred_df.to_csv('test_preds.csv')","metadata":{"_uuid":"07d648eb-b1bf-40bf-a933-bc293049eb78","_cell_guid":"90ba58f7-f84f-44d7-ab65-b40d2a3b1e02","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-14T14:28:18.326275Z","iopub.execute_input":"2021-09-14T14:28:18.326539Z","iopub.status.idle":"2021-09-14T14:28:18.39369Z","shell.execute_reply.started":"2021-09-14T14:28:18.326511Z","shell.execute_reply":"2021-09-14T14:28:18.392227Z"},"trusted":true},"execution_count":null,"outputs":[]}]}