{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":37333,"databundleVersionId":3949526,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:12:39.232336Z","iopub.execute_input":"2024-11-29T13:12:39.232893Z","iopub.status.idle":"2024-11-29T13:12:40.986721Z","shell.execute_reply.started":"2024-11-29T13:12:39.232854Z","shell.execute_reply":"2024-11-29T13:12:40.985815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip uninstall -y lightning-flash pytorch-lightning torchvision torch lightning-bolts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:12:40.988512Z","iopub.execute_input":"2024-11-29T13:12:40.989164Z","iopub.status.idle":"2024-11-29T13:12:43.671639Z","shell.execute_reply.started":"2024-11-29T13:12:40.98912Z","shell.execute_reply":"2024-11-29T13:12:43.670729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install torch==1.13.1 torchvision==0.14.1\n!pip install lightning-flash","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:12:43.672985Z","iopub.execute_input":"2024-11-29T13:12:43.673261Z","iopub.status.idle":"2024-11-29T13:13:26.254473Z","shell.execute_reply.started":"2024-11-29T13:12:43.673232Z","shell.execute_reply":"2024-11-29T13:13:26.253552Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n# 设置数据集路径\nDATASET_FOLDER = \"/kaggle/input/mayo-clinic-strip-ai/\"\nDATASET_SMALL_FOLDER = \"/kaggle/input/newtrain\"\n\n# 验证路径是否存在\nif not os.path.exists(DATASET_FOLDER):\n    print(f\"警告: {DATASET_FOLDER} 不存在\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:13:26.256786Z","iopub.execute_input":"2024-11-29T13:13:26.25712Z","iopub.status.idle":"2024-11-29T13:13:26.263735Z","shell.execute_reply.started":"2024-11-29T13:13:26.257082Z","shell.execute_reply":"2024-11-29T13:13:26.262842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport keras.backend as K #to define custom loss function\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom pprint import pprint\nfrom collections import defaultdict\nimport openslide\nfrom openslide import OpenSlide\n\nfrom glob import glob\n\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense\nfrom tensorflow.keras.layers import GlobalMaxPooling2D\nfrom keras.models import load_model\n\nprint(keras.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:13:26.264675Z","iopub.execute_input":"2024-11-29T13:13:26.264886Z","iopub.status.idle":"2024-11-29T13:13:26.281424Z","shell.execute_reply.started":"2024-11-29T13:13:26.264864Z","shell.execute_reply":"2024-11-29T13:13:26.280597Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\n\ntrain_df = pd.read_csv('../input/mayo-clinic-strip-ai/train.csv')\ntest_df  = pd.read_csv('../input/mayo-clinic-strip-ai/test.csv')\n\n# Specify patient_ids to remove\npatient_ids_to_remove = ['006388', '008e5c', '00c058', '01adc5']\n\n# Filter out the rows with specified patient_ids\ntrain_df = train_df[~train_df['patient_id'].isin(patient_ids_to_remove)].reset_index(drop=True)\ntrain_df = train_df.drop_duplicates(subset=['patient_id'])\n\ndf1 = train_df[train_df['label'] == 'CE']\ndf2 = train_df[train_df['label'] == 'LAA']\n#adjust n to change number of CE data\nsampled= df1.sample(n=200, random_state=42)\ntrain_df = pd.concat([sampled, df2],ignore_index=True)\n# Print the cleaned DataFrame\nprint(\"Cleaned DataFrame:\")\nprint(train_df.head())\ntrain_df['label'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:13:26.282464Z","iopub.execute_input":"2024-11-29T13:13:26.282723Z","iopub.status.idle":"2024-11-29T13:13:26.317004Z","shell.execute_reply.started":"2024-11-29T13:13:26.282696Z","shell.execute_reply":"2024-11-29T13:13:26.316257Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['patient_id'].nunique","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:13:26.317965Z","iopub.execute_input":"2024-11-29T13:13:26.318321Z","iopub.status.idle":"2024-11-29T13:13:26.324941Z","shell.execute_reply.started":"2024-11-29T13:13:26.31828Z","shell.execute_reply":"2024-11-29T13:13:26.324119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain_images = glob(\"/kaggle/input/mayo-clinic-strip-ai/train/*\")\ntest_images = glob(\"/kaggle/input/mayo-clinic-strip-ai/test/*\")\nother_images = glob(\"/kaggle/input/mayo-clinic-strip-ai/other/*\")\nprint(f\"Number of images in a training set: {len(train_images)}\")\nprint(f\"Number of images in a training set: {len(test_images)}\")\nprint(f\"Number of other: {len(other_images)}\")\n\n# Filtering out images based on the cleaned patient_ids in train_df\nimages_to_remove = [\n    '/kaggle/input/mayo-clinic-strip-ai/train/006388_0.tif',\n    '/kaggle/input/mayo-clinic-strip-ai/train/008e5c_0.tif',\n    '/kaggle/input/mayo-clinic-strip-ai/train/00c058_0.tif',\n    '/kaggle/input/mayo-clinic-strip-ai/train/01adc5_0.tif',\n]\n\n# Remove images associated with the patient_ids\ntrain_images = [img for img in train_images if img not in images_to_remove]\n\n# Check the total number of images after deletion\ntotal_images_after_deletion = len(train_images)\nprint(\"Total number of images after deletion:\", total_images_after_deletion)\n\n# Print the paths of the cleaned list of images\nprint(\"First 5 image paths after deletion:\")\nprint(train_images[:5])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:13:26.325878Z","iopub.execute_input":"2024-11-29T13:13:26.32618Z","iopub.status.idle":"2024-11-29T13:13:26.34014Z","shell.execute_reply.started":"2024-11-29T13:13:26.326132Z","shell.execute_reply":"2024-11-29T13:13:26.339306Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[\"file_path\"] = train_df[\"image_id\"].apply(lambda x: \"../input/mayo-clinic-strip-ai/train/\" + x + \".tif\")\ntest_df[\"file_path\"]  = test_df[\"image_id\"].apply(lambda x: \"../input/mayo-clinic-strip-ai/test/\" + x + \".tif\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:13:26.341235Z","iopub.execute_input":"2024-11-29T13:13:26.341884Z","iopub.status.idle":"2024-11-29T13:13:26.350597Z","shell.execute_reply.started":"2024-11-29T13:13:26.341846Z","shell.execute_reply":"2024-11-29T13:13:26.349867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# labelling CE class as 1 and LAA as 0\ntrain_df[\"target\"] = train_df[\"label\"].apply(lambda x : 1 if x==\"CE\" else 0)\ntrain_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:13:26.35324Z","iopub.execute_input":"2024-11-29T13:13:26.353519Z","iopub.status.idle":"2024-11-29T13:13:26.367283Z","shell.execute_reply.started":"2024-11-29T13:13:26.353492Z","shell.execute_reply":"2024-11-29T13:13:26.366484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ntrain_df[\"file_path\"] = train_df[\"image_id\"].apply(lambda x: \"../input/mayo-clinic-strip-ai/train/\" + x + \".tif\")\n\ndef preprocess(image_path):\n    slide=OpenSlide(image_path)\n    region= (2500,2500)    \n    size  = (5000, 5000)\n    image = slide.read_region(region, 0, size)\n    image = image.resize((128, 128))\n    image = np.array(image)    \n    return image\n\nX_train=[]\nfor i in tqdm(train_df['file_path']):\n    x1=preprocess(i)\n    X_train.append(x1)\n\nY_train=[]    \nY_train=train_df['target']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:13:26.368214Z","iopub.execute_input":"2024-11-29T13:13:26.368462Z","iopub.status.idle":"2024-11-29T13:26:59.558004Z","shell.execute_reply.started":"2024-11-29T13:13:26.368437Z","shell.execute_reply":"2024-11-29T13:26:59.557115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train=np.array(X_train)\nX_train=X_train/255.0\nY_train = np.array(Y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:26:59.559292Z","iopub.execute_input":"2024-11-29T13:26:59.55959Z","iopub.status.idle":"2024-11-29T13:26:59.631872Z","shell.execute_reply.started":"2024-11-29T13:26:59.559555Z","shell.execute_reply":"2024-11-29T13:26:59.631131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install tensorflow numpy pandas tqdm\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:26:59.973649Z","iopub.execute_input":"2024-11-29T13:26:59.974015Z","iopub.status.idle":"2024-11-29T13:27:08.719825Z","shell.execute_reply.started":"2024-11-29T13:26:59.973975Z","shell.execute_reply":"2024-11-29T13:27:08.718889Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.applications import EfficientNetB6\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Dropout, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:27:08.721447Z","iopub.execute_input":"2024-11-29T13:27:08.721831Z","iopub.status.idle":"2024-11-29T13:27:08.728405Z","shell.execute_reply.started":"2024-11-29T13:27:08.72179Z","shell.execute_reply":"2024-11-29T13:27:08.727357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Y_train shape:\", Y_train.shape)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:28:33.913825Z","iopub.execute_input":"2024-11-29T13:28:33.914542Z","iopub.status.idle":"2024-11-29T13:28:33.918928Z","shell.execute_reply.started":"2024-11-29T13:28:33.914507Z","shell.execute_reply":"2024-11-29T13:28:33.918014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Unique values in Y_train:\", np.unique(Y_train))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:28:44.599091Z","iopub.execute_input":"2024-11-29T13:28:44.599442Z","iopub.status.idle":"2024-11-29T13:28:44.604967Z","shell.execute_reply.started":"2024-11-29T13:28:44.599413Z","shell.execute_reply":"2024-11-29T13:28:44.604129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical\n\n# 将整数标签转换为 one-hot 编码\nY_train = to_categorical(Y_train, num_classes=2)\n\nprint(\"Y_train shape after one-hot encoding:\", Y_train.shape)\n# 输出: (373, 2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:29:28.371021Z","iopub.execute_input":"2024-11-29T13:29:28.371968Z","iopub.status.idle":"2024-11-29T13:29:28.3782Z","shell.execute_reply.started":"2024-11-29T13:29:28.37193Z","shell.execute_reply":"2024-11-29T13:29:28.377205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_batch, Y_batch = next(datagen.flow(X_train, Y_train, batch_size=32))\n\nprint(\"X_batch shape:\", X_batch.shape)\nprint(\"Y_batch shape:\", Y_batch.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:31:40.336239Z","iopub.execute_input":"2024-11-29T13:31:40.336954Z","iopub.status.idle":"2024-11-29T13:31:40.527973Z","shell.execute_reply.started":"2024-11-29T13:31:40.336921Z","shell.execute_reply":"2024-11-29T13:31:40.527016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom tensorflow.keras.utils import to_categorical\n\n\n\n# 数据增强\ndatagen = ImageDataGenerator(\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    fill_mode='nearest'\n)\n\n# 生成增强数据\ntrain_generator = datagen.flow(\n    X_train, \n    Y_train, \n    batch_size=32  # 去掉 class_mode\n)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:32:06.631646Z","iopub.execute_input":"2024-11-29T13:32:06.631982Z","iopub.status.idle":"2024-11-29T13:32:06.673316Z","shell.execute_reply.started":"2024-11-29T13:32:06.631953Z","shell.execute_reply":"2024-11-29T13:32:06.672557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.applications import EfficientNetB6\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Dropout, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.utils import to_categorical\n\n\n\n# 加载 EfficientNet-B6 的预训练模型\nbase_model = EfficientNetB6(weights=None, include_top=False, input_shape=(128, 128, 4))\n\n# 冻结预训练模型的卷积层\nbase_model.trainable = False\n\n# 添加自定义分类头\nx = base_model.output\nx = GlobalAveragePooling2D()(x)\nx = Dropout(0.5)(x)\nx = Dense(256, activation='relu')(x)\nx = Dropout(0.5)(x)\npredictions = Dense(Y_train.shape[1], activation='softmax')(x)  # 输出层\n\n# 定义完整模型\nmodel = Model(inputs=base_model.input, outputs=predictions)\n\n# 编译模型\nmodel.compile(optimizer=Adam(learning_rate=0.001), loss='categorical_crossentropy', metrics=['accuracy'])\n\n# 打印模型结构\nmodel.summary()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:32:40.564929Z","iopub.execute_input":"2024-11-29T13:32:40.565621Z","iopub.status.idle":"2024-11-29T13:32:43.01584Z","shell.execute_reply.started":"2024-11-29T13:32:40.565584Z","shell.execute_reply":"2024-11-29T13:32:43.014882Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n\n# 定义回调函数\nearly_stopping = EarlyStopping(monitor='loss', patience=5, restore_best_weights=True)  # 监控训练集的 loss\nmodel_checkpoint = ModelCheckpoint('efficientnet_b6_best_model.keras', save_best_only=True, monitor='loss')  # 保存基于训练集 loss 的最优模型\n\n# 开始训练\nhistory = model.fit(\n    train_generator,  # 训练数据生成器\n    epochs=20,        # 训练轮数\n    callbacks=[early_stopping, model_checkpoint]\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:33:04.995188Z","iopub.execute_input":"2024-11-29T13:33:04.995933Z","iopub.status.idle":"2024-11-29T13:36:50.680347Z","shell.execute_reply.started":"2024-11-29T13:33:04.995897Z","shell.execute_reply":"2024-11-29T13:36:50.679573Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntest1=[]\nfor i in test_df['file_path']:\n    x1=preprocess(i)\n    test1.append(x1)\n    print(i)\n    \ntest1=np.array(test1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T14:17:44.155602Z","iopub.execute_input":"2024-11-29T14:17:44.156368Z","iopub.status.idle":"2024-11-29T14:18:05.176643Z","shell.execute_reply.started":"2024-11-29T14:17:44.156332Z","shell.execute_reply":"2024-11-29T14:18:05.17574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 假设类别索引与名称的映射\nclass_names = ['CE', 'LAA']  # 0 对应 'CE'，1 对应 'LAA'\n\n# 加载保存的模型\nfrom tensorflow.keras.models import load_model\nmodel = load_model('efficientnet_b6_best_model.keras')\nprint(\"Model loaded successfully!\")\n\n# 确保 test1 数据已经准备好并归一化（如果需要）\ntest1 = test1 / 255.0  # 如果像素值范围是 0-255，进行归一化\n\n\n\n# 获取每张图片的预测类别索引\npredicted_classes = predictions.argmax(axis=1)  # 获取每张图片的预测类别索引\n\n# 打印每张图片的预测结果\nfor i, predicted_class in enumerate(predicted_classes):\n    print(f\"Image {i+1}: Predicted Class = {class_names[predicted_class]}\")\n\n# 可视化这 4 张图片及其预测结果\nimport matplotlib.pyplot as plt\n\nfor i in range(len(selected_images)):\n    plt.imshow(selected_images[i])  # 显示图片\n    plt.title(f\"Predicted: {class_names[predicted_classes[i]]}\")  # 显示预测类别\n    plt.axis('off')\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T14:43:11.850084Z","iopub.execute_input":"2024-11-29T14:43:11.850423Z","iopub.status.idle":"2024-11-29T14:43:35.09582Z","shell.execute_reply.started":"2024-11-29T14:43:11.850395Z","shell.execute_reply":"2024-11-29T14:43:35.094915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(type(Y_train))  # 检查数据类型\nprint(Y_train.shape)  # 检查形状\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T13:27:08.824943Z","iopub.status.idle":"2024-11-29T13:27:08.825408Z","shell.execute_reply.started":"2024-11-29T13:27:08.825178Z","shell.execute_reply":"2024-11-29T13:27:08.825202Z"}},"outputs":[],"execution_count":null}]}