{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":37333,"databundleVersionId":3949526,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-27T12:55:04.09578Z","iopub.execute_input":"2024-11-27T12:55:04.096179Z","iopub.status.idle":"2024-11-27T12:55:06.740522Z","shell.execute_reply.started":"2024-11-27T12:55:04.096135Z","shell.execute_reply":"2024-11-27T12:55:06.739569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip uninstall -y lightning-flash pytorch-lightning torchvision torch lightning-bolts\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T12:55:06.741722Z","iopub.execute_input":"2024-11-27T12:55:06.742086Z","iopub.status.idle":"2024-11-27T12:55:25.346872Z","shell.execute_reply.started":"2024-11-27T12:55:06.742043Z","shell.execute_reply":"2024-11-27T12:55:25.344878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install torch==1.13.1 torchvision==0.14.1\n!pip install lightning-flash\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T12:55:25.349888Z","iopub.execute_input":"2024-11-27T12:55:25.350287Z","iopub.status.idle":"2024-11-27T12:57:01.038031Z","shell.execute_reply.started":"2024-11-27T12:55:25.350246Z","shell.execute_reply":"2024-11-27T12:57:01.037068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n# 设置数据集路径\nDATASET_FOLDER = \"/kaggle/input/mayo-clinic-strip-ai/\"\n\n\n# 验证路径是否存在\nif not os.path.exists(DATASET_FOLDER):\n    print(f\"警告: {DATASET_FOLDER} 不存在\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T12:59:04.745462Z","iopub.execute_input":"2024-11-27T12:59:04.746171Z","iopub.status.idle":"2024-11-27T12:59:04.753192Z","shell.execute_reply.started":"2024-11-27T12:59:04.746119Z","shell.execute_reply":"2024-11-27T12:59:04.752168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path_csv = os.path.join(DATASET_FOLDER, \"train.csv\")\ndf_train = pd.read_csv(path_csv)\ndisplay(df_train.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T12:59:08.418869Z","iopub.execute_input":"2024-11-27T12:59:08.419241Z","iopub.status.idle":"2024-11-27T12:59:08.435008Z","shell.execute_reply.started":"2024-11-27T12:59:08.419207Z","shell.execute_reply":"2024-11-27T12:59:08.434097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom tqdm.auto import tqdm\nfrom joblib import Parallel, delayed\nimport matplotlib.pyplot as plt\nimport glob\nimport os\nfrom PIL import Image\nimport torchvision.transforms as T\n\nDATASET_FOLDER = \"/kaggle/input/mayo-clinic-strip-ai\"\nTRAIN_IMAGE_PATH = os.path.join(DATASET_FOLDER, \"train\")\n\ndef _color_means(img_path):\n    \"\"\"计算图像的颜色均值和标准差\"\"\"\n    try:\n        # 使用PIL打开图像\n        img = Image.open(img_path)\n        \n        # 使用transforms进行预处理\n        transform = T.Compose([\n            T.CenterCrop(size=(800, 800)),  # 先裁剪到固定大小\n            T.ToTensor()  # 转换为tensor并自动归一化到[0,1]\n        ])\n        \n        img = transform(img)\n        \n        # 计算均值和标准差\n        clr_mean = {i: float(img[i].mean()) for i in range(3)}\n        clr_std = {i: float(img[i].std()) for i in range(3)}\n        \n        return clr_mean, clr_std\n    except Exception as e:\n        print(f\"处理图像 {img_path} 时出错: {str(e)}\")\n        return None\n\n# 获取所有训练图像路径\nimages = glob.glob(os.path.join(TRAIN_IMAGE_PATH, \"*0.tif\"))\nprint(f\"找到 {len(images)} 个训练图像文件\")\n\nif len(images) > 0:\n    # 使用tqdm显示进度\n    clr_mean_std = Parallel(n_jobs=os.cpu_count())(\n        delayed(_color_means)(fn) for fn in tqdm(images[::10], desc=\"处理图像\")\n    )\n    \n    # 显示结果\n    valid_results = [r for r in clr_mean_std if r is not None]\n    if valid_results:\n        print(\"\\n处理完成！\")\n        print(f\"成功处理 {len(valid_results)} 个图像\")\n        print(\"\\n示例结果:\")\n        print(valid_results[0])\n        \n        # 计算所有通道的平均值和标准差\n        means = np.array([[r[0][i] for i in range(3)] for r in valid_results])\n        stds = np.array([[r[1][i] for i in range(3)] for r in valid_results])\n        \n        print(\"\\n各通道的平均值:\")\n        print(means.mean(axis=0))\n        print(\"\\n各通道的标准差:\")\n        print(stds.mean(axis=0))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T12:59:15.229953Z","iopub.execute_input":"2024-11-27T12:59:15.230601Z","iopub.status.idle":"2024-11-27T12:59:22.884377Z","shell.execute_reply.started":"2024-11-27T12:59:15.230566Z","shell.execute_reply":"2024-11-27T12:59:22.883322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\nfrom joblib import Parallel, delayed\nimport glob\nimport os\nfrom PIL import Image\n\n# 使用正确的训练图像路径\nDATASET_FOLDER = \"/kaggle/input/mayo-clinic-strip-ai\"\nTRAIN_IMAGE_PATH = os.path.join(DATASET_FOLDER, \"train\")\n\ndef _color_means(img_path):\n    \"\"\"计算图像的颜色均值和标准差\"\"\"\n    try:\n        # 使用PIL打开图像\n        img = Image.open(img_path)\n        \n        # 获取图像尺寸\n        w, h = img.size\n        \n        # 计算缩放比例，确保最大边长不超过1024像素\n        scale = min(1024/w, 1024/h, 1)\n        if scale < 1:\n            new_w, new_h = int(w*scale), int(h*scale)\n            img = img.resize((new_w, new_h))\n        \n        # 转换为numpy数组\n        img = np.array(img).astype(np.float32) / 255.0\n        \n        # 计算均值和标准差\n        clr_mean = {i: np.mean(img[..., i]) for i in range(3)}\n        clr_std = {i: np.std(img[..., i]) for i in range(3)}\n        \n        return clr_mean, clr_std\n    except Exception as e:\n        print(f\"处理图像 {img_path} 时出错: {str(e)}\")\n        return None\n\n# 获取所有训练图像路径\nimages = glob.glob(os.path.join(TRAIN_IMAGE_PATH, \"*0.tif\"))\nprint(f\"找到 {len(images)} 个训练图像文件\")\n\nif len(images) > 0:\n    # 计算所有图像的颜色统计特征\n    clr_mean_std = Parallel(n_jobs=os.cpu_count())(\n        delayed(_color_means)(fn) for fn in tqdm(images[::10], desc=\"处理图像\")\n    )\n    \n    # 移除None值\n    clr_mean_std = [x for x in clr_mean_std if x is not None]\n    \n    if len(clr_mean_std) > 0:\n        # 处理RGB图像数据\n        means_df = pd.DataFrame([c[0] for c in clr_mean_std])\n        stds_df = pd.DataFrame([c[1] for c in clr_mean_std])\n        \n        # 计算描述性统计\n        img_color_mean = means_df.describe()\n        print(\"\\n颜色均值的描述性统计：\")\n        display(img_color_mean.T)\n        \n        img_color_std = stds_df.describe()\n        print(\"\\n颜色标准差的描述性统计：\")\n        display(img_color_std.T)\n        \n        # 获取平均值\n        img_color_mean = list(img_color_mean.T[\"mean\"])\n        img_color_std = list(img_color_std.T[\"mean\"])\n        print(\"\\n最终的颜色统计特征：\")\n        print(\"颜色均值:\", img_color_mean)\n        print(\"颜色标准差:\", img_color_std)\n    else:\n        print(\"没有成功处理任何图像！\")\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T12:59:36.794232Z","iopub.execute_input":"2024-11-27T12:59:36.795072Z","iopub.status.idle":"2024-11-27T12:59:43.29767Z","shell.execute_reply.started":"2024-11-27T12:59:36.795035Z","shell.execute_reply":"2024-11-27T12:59:43.296818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image\nimport numpy as np\n\n# 设置PIL可处理的最大图像像素数\nImage.MAX_IMAGE_PIXELS = 25_000_000_000\n\ndef prune_image_rows_cols(im, mask, thr=0.990):\n    \"\"\"\n    裁剪图像中的空白区域\n    \"\"\"\n    if im is None or mask is None:\n        return im\n    \n    # 删除空白列\n    for l in reversed(range(im.shape[1])):\n        if (np.sum(mask[:, l]) / float(mask.shape[0])) > thr:\n            im = np.delete(im, l, 1)\n    # 删除空白行\n    for l in reversed(range(im.shape[0])):\n        if (np.sum(mask[l, :]) / float(mask.shape[1])) > thr:\n            im = np.delete(im, l, 0)\n    return im\n\ndef mask_median(im, val=255):\n    \"\"\"\n    基于中值的背景分割\n    \"\"\"\n    if im is None:\n        return None, None\n    \n    try:\n        masks = [None] * 3\n        for c in range(3):\n            masks[c] = im[..., c] >= np.median(im[:, :, c]) - 5\n        mask = np.logical_and.reduce(masks)  # 使用reduce更简洁\n        im[mask, :] = val\n        return im, mask\n    except Exception as e:\n        print(f\"mask_median处理出错: {str(e)}\")\n        return None, None\n\ndef image_load_scale_norm(img_path, prune_thr=0.990, bg_val=255):\n    \"\"\"\n    加载、缩放和标准化图像\n    \n    参数:\n    - img_path: 图像文件路径\n    - prune_thr: 裁剪阈值\n    - bg_val: 背景值\n    \n    返回:\n    - 处理后的PIL图像对象，或None（如果处理失败）\n    \"\"\"\n    try:\n        # 加载图像\n        img = Image.open(img_path)\n        \n        # 检查图像尺寸\n        if (img.width * img.height) > 1_500_000_000:\n            print(f\"图像太大: {img.width}x{img.height}\")\n            return None\n        \n        # 第一次缩放\n        scale = max(1, min(img.height / 2e3, img.width / 2e3))\n        if scale > 1:\n            tmp_size = (int(img.width / scale), int(img.height / scale))\n            img = img.resize(tmp_size, Image.Resampling.BILINEAR)\n        \n        # 背景处理和裁剪\n        im_array = np.array(img)\n        if im_array.ndim != 3:\n            print(\"不支持的图像格式（需要RGB图像）\")\n            return None\n            \n        im_array, mask = mask_median(im_array, val=bg_val)\n        if im_array is None:\n            return None\n            \n        im_array = prune_image_rows_cols(im_array, mask, thr=prune_thr)\n        \n        # 转回PIL图像\n        img = Image.fromarray(im_array)\n        \n        # 最终缩放\n        scale = max(1, min(img.height / 1e3, img.width / 1e3))\n        if scale > 1:\n            final_size = (int(img.width / scale), int(img.height / scale))\n            img = img.resize(final_size, Image.LANCZOS)\n        \n        return img\n        \n    except Exception as e:\n        print(f\"处理图像 {img_path} 时出错: {str(e)}\")\n        return None\n\n# 使用示例\ndef process_image(image_path):\n    try:\n        processed_img = image_load_scale_norm(image_path)\n        if processed_img is not None:\n            print(f\"成功处理图像: {image_path}\")\n            print(f\"处理后尺寸: {processed_img.size}\")\n            return processed_img\n        else:\n            print(f\"处理图像失败: {image_path}\")\n            return None\n    except Exception as e:\n        print(f\"处理出错: {str(e)}\")\n        return None\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T04:46:55.647443Z","iopub.execute_input":"2024-11-28T04:46:55.647772Z","iopub.status.idle":"2024-11-28T04:46:55.665421Z","shell.execute_reply.started":"2024-11-28T04:46:55.647745Z","shell.execute_reply":"2024-11-28T04:46:55.664574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\nfrom tqdm.auto import tqdm\nimport os\nfrom PIL import Image\nimport numpy as np\n\ndef image_load_scale_norm(img_path, max_size=2000):\n    try:\n        # 加载图像\n        img = Image.open(img_path)\n        \n        # 计算缩放比例\n        scale = max(img.width / max_size, img.height / max_size)\n        if scale > 1:\n            new_width = int(img.width / scale)\n            new_height = int(img.height / scale)\n            print(f\"缩放图像 {os.path.basename(img_path)} 从 {img.width}x{img.height} 到 {new_width}x{new_height}\")\n            img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)\n\n        # 转换为numpy数组进行处理\n        im = np.array(img)\n        \n        # 原有的处理逻辑\n        im, mask = mask_median(im, 255)\n        if im is None:\n            return None\n        \n        im = prune_image_rows_cols(im, mask, 0.990)\n        if im is None:\n            return None\n            \n        return Image.fromarray(im)\n        \n    except Exception as e:\n        print(f\"处理图像 {os.path.basename(img_path)} 时出错: {str(e)}\")\n        return None\n\nls_imgs_tif = glob.glob(os.path.join(DATASET_FOLDER, \"test\", \"*0.tif\"))\nnames = [os.path.splitext(os.path.basename(p))[0] for p in ls_imgs_tif]\npatient_ids = set([n.split(\"_\")[0] for n in names])\nprint(f\"找到 {len(patient_ids)} 个病人ID：\")\nprint(patient_ids)\n\nos.makedirs(\"test_images\", exist_ok=True)\n\nprocessed_count = 0\ntotal_images = len(ls_imgs_tif)\n\nfor img_path in tqdm(ls_imgs_tif):\n    name, _ = os.path.splitext(os.path.basename(img_path))\n    img = image_load_scale_norm(img_path)\n    if not img:\n        print(f\"missing: {name}\")\n        continue\n    img.save(os.path.join(\"test_images\", f\"{name}.png\"))\n    processed_count += 1\n    del img\n    gc.collect()\n\nprint(f\"\\n处理完成！\")\nprint(f\"成功处理: {processed_count}/{total_images} 张图像\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T13:00:29.694177Z","iopub.execute_input":"2024-11-27T13:00:29.694786Z","iopub.status.idle":"2024-11-27T13:02:18.477521Z","shell.execute_reply.started":"2024-11-27T13:00:29.694751Z","shell.execute_reply":"2024-11-27T13:02:18.476634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 检查文件夹内容\nprint(f\"DATASET_FOLDER路径: {DATASET_FOLDER}\")\nall_files = os.listdir(os.path.join(DATASET_FOLDER, \"test\"))\nprint(f\"test文件夹中所有文件: {all_files}\")\n\n# 检查.tif文件\nprint(f\"\\n找到的.tif文件:\")\nfor tif_file in ls_imgs_tif:\n    print(tif_file)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T13:02:18.512191Z","iopub.execute_input":"2024-11-27T13:02:18.512437Z","iopub.status.idle":"2024-11-27T13:02:18.530452Z","shell.execute_reply.started":"2024-11-27T13:02:18.512412Z","shell.execute_reply":"2024-11-27T13:02:18.529679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from dataclasses import dataclass\nfrom torchvision import transforms as T\nfrom typing import Tuple, Callable\nfrom flash.core.data.io.input_transform import InputTransform\nfrom flash.core.utilities.stages import RunningStage\n\n@dataclass\nclass ImageClassifInputTransform(InputTransform):\n    image_size: Tuple[int, int] = (256, 256)\n    color_mean: Tuple[float, float, float] = (0.947, 0.881, 0.863)\n    color_std: Tuple[float, float, float] = (0.093, 0.201, 0.245)\n\n    def __init__(self, \n                 image_size: Tuple[int, int] = (256, 256),\n                 color_mean: Tuple[float, float, float] = (0.947, 0.881, 0.863),\n                 color_std: Tuple[float, float, float] = (0.093, 0.201, 0.245)):\n        super().__init__(running_stage=RunningStage.TRAINING)\n        self.image_size = image_size\n        self.color_mean = color_mean\n        self.color_std = color_std\n\n    def train_input_per_sample_transform(self) -> Callable:\n        return T.Compose([\n            T.ToTensor(),\n            T.RandomCrop(size=(800, 800), pad_if_needed=True),\n            T.GaussianBlur(kernel_size=5, sigma=(0.5, 4)),\n            T.Resize(self.image_size),\n            T.RandomHorizontalFlip(),\n            T.RandomVerticalFlip(),\n            T.RandomAffine(degrees=30, translate=(0.15, 0.15)),\n            T.Normalize(self.color_mean, self.color_std),\n        ])\n\n    def input_per_sample_transform(self) -> Callable:\n        return T.Compose([\n            T.ToTensor(),\n            T.CenterCrop(size=(800, 800)),\n            T.Resize(self.image_size),\n            T.Normalize(self.color_mean, self.color_std),\n        ])\n\n# 使用示例\ntransform = ImageClassifInputTransform()\ntrain_transform = transform.train_input_per_sample_transform()\nval_transform = transform.input_per_sample_transform()\n\ndef process_image(img_path, is_training=True):\n    try:\n        img = Image.open(img_path)\n        if is_training:\n            img = train_transform(img)\n        else:\n            img = val_transform(img)\n        return img\n    except Exception as e:\n        print(f\"处理图像 {img_path} 时出错: {str(e)}\")\n        return None\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T13:03:08.068567Z","iopub.execute_input":"2024-11-27T13:03:08.06888Z","iopub.status.idle":"2024-11-27T13:03:12.210908Z","shell.execute_reply.started":"2024-11-27T13:03:08.068854Z","shell.execute_reply":"2024-11-27T13:03:12.209409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from dataclasses import dataclass\nfrom torchvision import transforms as T\nfrom typing import Tuple, Callable\nfrom flash.core.data.io.input_transform import InputTransform\nfrom flash.core.utilities.stages import RunningStage\nfrom PIL import Image\n\n@dataclass\nclass ImageClassifInputTransform(InputTransform):\n    image_size: Tuple[int, int] = (256, 256)\n    color_mean: Tuple[float, float, float] = (0.947, 0.881, 0.863)\n    color_std: Tuple[float, float, float] = (0.093, 0.201, 0.245)\n\n    def __init__(self, \n                 image_size: Tuple[int, int] = (256, 256),\n                 color_mean: Tuple[float, float, float] = (0.947, 0.881, 0.863),\n                 color_std: Tuple[float, float, float] = (0.093, 0.201, 0.245)):\n        # 移除 running_stage 参数\n        super().__init__()\n        self.image_size = image_size\n        self.color_mean = color_mean\n        self.color_std = color_std\n        self.running_stage = RunningStage.TRAINING  # 如果需要，可以这样设置\n\n    def train_input_per_sample_transform(self) -> Callable:\n        return T.Compose([\n            T.ToTensor(),\n            T.RandomCrop(size=(800, 800), pad_if_needed=True),\n            T.GaussianBlur(kernel_size=5, sigma=(0.5, 4)),\n            T.Resize(self.image_size),\n            T.RandomHorizontalFlip(),\n            T.RandomVerticalFlip(),\n            T.RandomAffine(degrees=30, translate=(0.15, 0.15)),\n            T.Normalize(self.color_mean, self.color_std),\n        ])\n\n    def input_per_sample_transform(self) -> Callable:\n        return T.Compose([\n            T.ToTensor(),\n            T.CenterCrop(size=(800, 800)),\n            T.Resize(self.image_size),\n            T.Normalize(self.color_mean, self.color_std),\n        ])\n\ndef process_image(img_path, transform, is_training=True):\n    try:\n        img = Image.open(img_path)\n        if is_training:\n            img = transform.train_input_per_sample_transform()(img)\n        else:\n            img = transform.input_per_sample_transform()(img)\n        return img\n    except Exception as e:\n        print(f\"处理图像 {img_path} 时出错: {str(e)}\")\n        return None\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T13:03:21.749154Z","iopub.execute_input":"2024-11-27T13:03:21.749517Z","iopub.status.idle":"2024-11-27T13:03:22.010143Z","shell.execute_reply.started":"2024-11-27T13:03:21.749484Z","shell.execute_reply":"2024-11-27T13:03:22.008718Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\n\n# 定义路径\nDATASET_FOLDER = \"../input/mayo-clinic-strip-ai\"\nTRAIN_IMAGE_PATH = os.path.join(DATASET_FOLDER, \"train\")  # 修改为正确的训练图像文件夹路径\n\n# 读取训练集的标签文件\ndf_train = pd.read_csv(os.path.join(DATASET_FOLDER, \"train.csv\"))\n\n# 添加图像文件名列\ndf_train['img_name'] = df_train['image_id'].apply(lambda n: f\"{n}.tif\")\n\n# 显示数据集的前5行\nprint(\"训练集数据预览:\")\ndisplay(df_train.head())\nprint(\"\\n原始训练集数量:\", len(df_train))\n\n# 检查图像文件是否存在\nmissing = df_train['img_name'].apply(lambda n: not os.path.isfile(os.path.join(TRAIN_IMAGE_PATH, n)))\ndf_train = df_train[~missing]\nprint(\"清理后训练集数量:\", len(df_train))\n\n# 获取所有有效的训练图像路径\nimages = [os.path.join(TRAIN_IMAGE_PATH, img_name) for img_name in df_train['img_name']]\n\n# 验证最终的图像列表\nprint(f\"找到 {len(images)} 个有效的训练图像文件\")\n\n# 显示几个示例图像路径\nprint(\"\\n示例图像路径:\")\nfor path in images[:3]:\n    print(path)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T13:03:27.347378Z","iopub.execute_input":"2024-11-27T13:03:27.34813Z","iopub.status.idle":"2024-11-27T13:03:28.355083Z","shell.execute_reply.started":"2024-11-27T13:03:27.348094Z","shell.execute_reply":"2024-11-27T13:03:28.35415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\n\n# 增加 PIL 的限制\nImage.MAX_IMAGE_PIXELS = None  # 禁用限制检查\n\n# 定义自定义数据集类\nclass CustomImageDataset(Dataset):\n    def __init__(self, df, img_dir, transform=None):\n        self.df = df\n        self.img_dir = img_dir\n        self.transform = transform\n        # 创建标签到数字的映射\n        self.label_map = {'CE': 0, 'LAA': 1}\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, self.df.iloc[idx]['img_name'])\n        image = Image.open(img_path)\n        \n        # 如果图像太大，先进行初步缩放\n        if image.size[0] > 1000 or image.size[1] > 1000:\n            image.thumbnail((1000, 1000), Image.LANCZOS)\n            \n        # 将标签转换为数字\n        label = self.label_map[self.df.iloc[idx]['label']]\n        label = torch.tensor(label, dtype=torch.long)\n        \n        if self.transform:\n            image = self.transform(image)\n            \n        return image, label\n\n# 定义路径和参数\nDATASET_FOLDER = \"../input/mayo-clinic-strip-ai\"\nTRAIN_IMAGE_PATH = os.path.join(DATASET_FOLDER, \"train\")\nTRANSFORM_PARAMS = {\n    \"image_size\": (528, 528),\n}\n\n# 读取训练集的标签文件\ndf_train = pd.read_csv(os.path.join(DATASET_FOLDER, \"train.csv\"))\ndf_train['img_name'] = df_train['image_id'].apply(lambda n: f\"{n}.tif\")\n\n# 定义图像转换\ntransform = transforms.Compose([\n    transforms.Resize(TRANSFORM_PARAMS[\"image_size\"]),\n    transforms.ToTensor(),\n])\n\n# 分割训练集和验证集\ntrain_df, val_df = train_test_split(df_train, test_size=0.1, random_state=42)\n\n# 创建数据集\ntrain_dataset = CustomImageDataset(\n    df=train_df,\n    img_dir=TRAIN_IMAGE_PATH,\n    transform=transform\n)\n\nval_dataset = CustomImageDataset(\n    df=val_df,\n    img_dir=TRAIN_IMAGE_PATH,\n    transform=transform\n)\n\n# 创建数据加载器\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=8,\n    shuffle=True,\n    num_workers=2\n)\n\nval_loader = DataLoader(\n    val_dataset,\n    batch_size=8,\n    shuffle=False,\n    num_workers=2\n)\n\n# 验证数据加载\nprint(\"数据集信息:\")\nprint(f\"训练集大小: {len(train_dataset)}\")\nprint(f\"验证集大小: {len(val_dataset)}\")\n\n# 获取一个批次的数据查看\ntrain_batch = next(iter(train_loader))\nprint(\"\\n批次信息:\")\nprint(f\"图片张量形状: {train_batch[0].shape}\")\nprint(f\"标签张量形状: {train_batch[1].shape}\")\nprint(f\"标签类别: {train_batch[1].unique().tolist()}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T06:38:22.468989Z","iopub.execute_input":"2024-11-18T06:38:22.469378Z","iopub.status.idle":"2024-11-18T06:39:39.582096Z","shell.execute_reply.started":"2024-11-18T06:38:22.46934Z","shell.execute_reply":"2024-11-18T06:39:39.58035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\n# 创建一个8x8的图像网格（2列，每列4行）\nfig, axarr = plt.subplots(ncols=2, nrows=4, figsize=(8, 16))\n\n# 用于记录每个类别已经显示的图片数量\nlb_counts = [0, 0]  # 对应两个类别：CE (0) 和 LAA (1)\n\n# 遍历数据加载器\nfor batch in train_loader:\n    images, labels = batch  # 获取图片和标签\n    \n    # 将图片从张量转换为numpy数组，并调整通道顺序\n    imgs = [img.permute(1, 2, 0).numpy() for img in images]\n    \n    # 遍历批次中的每张图片\n    for i in range(len(imgs)):\n        label = labels[i].item()  # 获取标签\n        count = lb_counts[label]  # 获取当前类别已显示的图片数量\n        \n        # 如果当前类别已经显示了4张图片，则跳过\n        if count >= 4:\n            continue\n            \n        # 显示图片\n        axarr[count, label].imshow(imgs[i])\n        axarr[count, label].set_title(f\"Label: {'CE' if label==0 else 'LAA'}\")\n        axarr[count, label].axis('off')  # 关闭坐标轴\n        \n        # 更新计数\n        lb_counts[label] += 1\n    \n    # 如果两个类别都已经显示了4张图片，则退出循环\n    if min(lb_counts) == 4:\n        break\n\n# 调整子图之间的间距\nplt.tight_layout()\nplt.show()\n\n# 打印每个类别显示的图片数量\nprint(\"显示的图片数量：\")\nprint(f\"CE (标签 0): {lb_counts[0]}张\")\nprint(f\"LAA (标签 1): {lb_counts[1]}张\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T06:39:56.675507Z","iopub.execute_input":"2024-11-18T06:39:56.675946Z","iopub.status.idle":"2024-11-18T06:40:58.829212Z","shell.execute_reply.started":"2024-11-18T06:39:56.675904Z","shell.execute_reply":"2024-11-18T06:40:58.828153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torchvision.models as models\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import StepLR\n\n# 加载预训练的 EfficientNet-B6\nmodel = models.efficientnet_b6(pretrained=True)\n\n# 如果需要修改最后的分类层来匹配你的类别数\nnum_classes = len(batch)  # 替换 your_labels 为你的实际标签列表\nmodel.classifier = torch.nn.Linear(model.classifier[1].in_features, num_classes)\n\n# 设置优化器\noptimizer = optim.Adamax(model.parameters(), lr=0.005)\n\n# 设置学习率调度器\nscheduler = StepLR(optimizer, step_size=150)\n\n# 如果有GPU则使用GPU\nif torch.cuda.is_available():\n    model = model.to('cuda')\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T06:41:38.711174Z","iopub.execute_input":"2024-11-18T06:41:38.711619Z","iopub.status.idle":"2024-11-18T06:41:41.523486Z","shell.execute_reply.started":"2024-11-18T06:41:38.711558Z","shell.execute_reply":"2024-11-18T06:41:41.522333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfrom glob import glob\n\n# 打印当前工作目录\nprint(\"当前工作目录:\", os.getcwd())\n\n# 列出当前目录下的所有文件夹\nprint(\"\\n当前目录下的文件夹:\")\nfor item in os.listdir():\n    if os.path.isdir(item):\n        print(item)\n\n# 列出当前目录下的所有PNG文件\nprint(\"\\n当前目录下的PNG文件:\")\npng_files = glob('*.png')\nfor file in png_files:\n    print(file)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T05:52:00.583172Z","iopub.execute_input":"2024-11-18T05:52:00.584026Z","iopub.status.idle":"2024-11-18T05:52:00.591239Z","shell.execute_reply.started":"2024-11-18T05:52:00.583981Z","shell.execute_reply":"2024-11-18T05:52:00.590357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torchvision import transforms\nfrom PIL import Image\nimport pandas as pd\nimport os\nfrom glob import glob\nimport numpy as np\n\n# 1. 设置图片路径\nimage_folder = '/kaggle/working/test_images'\nls_imgs_png = glob(os.path.join(image_folder, '*.png'))\n\nprint(f\"找到 {len(ls_imgs_png)} 张图片\")\nif len(ls_imgs_png) > 0:\n    print(\"示例图片路径:\", ls_imgs_png[0])\n\n# 2. 定义图像预处理转换\ntransform = transforms.Compose([\n    transforms.Resize((528, 528)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], \n                        std=[0.229, 0.224, 0.225])\n])\n\n# 3. 预测函数\ndef predict_images(model, image_paths, device):\n    model.eval()\n    predictions = []\n    \n    for img_path in image_paths:\n        img = Image.open(img_path).convert('RGB')\n        img_tensor = transform(img).unsqueeze(0).to(device)\n        \n        with torch.no_grad():\n            outputs = model(img_tensor)\n            probs = torch.nn.functional.softmax(outputs, dim=1)\n            # 交换预测结果的顺序\n            pred = probs.cpu().numpy()[0]\n            predictions.append([pred[1], pred[0]])  # 交换顺序\n    \n    return np.array(predictions)\n\n# 4. 进行预测\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = model.to(device)\npredictions = predict_images(model, ls_imgs_png, device)\n\n# 5. 创建结果列表\nresults = []\nlabels = ['CE', 'LAA']  # 标签顺序保持不变\n\nfor i, pred in enumerate(predictions):\n    result = {\n        'filename': os.path.basename(ls_imgs_png[i]),\n        'CE': pred[0],\n        'LAA': pred[1],\n        'predicted_class': labels[np.argmax(pred)],\n        'confidence': np.max(pred)\n    }\n    results.append(result)\n\n# 6. 创建DataFrame\ndf_pred = pd.DataFrame(results)\n\n# 7. 显示结果\nprint(\"\\n详细预测结果:\")\nfor _, row in df_pred.iterrows():\n    print(f\"图片: {row['filename']}\")\n    print(f\"预测类别: {row['predicted_class']}\")\n    print(f\"置信度: {row['confidence']:.2%}\")\n    print(f\"各类别概率: CE={row['CE']:.2%}, LAA={row['LAA']:.2%}\")\n    print()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T06:41:51.273043Z","iopub.execute_input":"2024-11-18T06:41:51.273798Z","iopub.status.idle":"2024-11-18T06:41:52.197168Z","shell.execute_reply.started":"2024-11-18T06:41:51.273756Z","shell.execute_reply":"2024-11-18T06:41:52.194998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# Sample data for submission DataFrame\ndata = {\n    \"patient_id\": [\"006388\", \"008e5c\", \"00c058\", \"01adc5\"],\n    \"CE\": [0.5172, 0.5167, 0.4978, 0.4628],\n    \"LAA\": [0.4628, 0.4833, 0.5022, 0.5372]\n}\nsub = pd.DataFrame(data)\n\n# Apply prediction logic\nsub[\"prediction\"] = sub[\"CE\"].apply(lambda x: \"CE\" if x >= 0.5 else \"LAA\")\nsub[\"actual\"] = [\"CE\", \"CE\", \"LAA\", \"LAA\"]\nprint(sub)\n\n# Weighted log loss function\ndef weighted_log_loss(y_true, y_pred, weights, N):\n    y_true = np.array(y_true)\n    y_pred = np.array(y_pred)\n    weights = np.array(weights)\n    N = np.array(N)\n\n    # Compute log loss for each class\n    log_loss_per_class = -np.sum(y_true / N[:, np.newaxis] * np.log(y_pred), axis=0)\n\n    # Weighted log loss\n    weighted_log_loss_value = np.sum(weights * log_loss_per_class) / np.sum(weights)\n\n    return weighted_log_loss_value\n\n# True labels (one-hot encoded)\ny_true = np.array([[1, 0],  # Patient 1: CE\n                   [1, 0],  # Patient 2: CE\n                   [0, 1],  # Patient 3: LAA\n                   [0, 1]]) # Patient 4: LAA\n\n# Sample N values\nN = [545, 545, 205, 205]\n\n# Clip y_pred to avoid log(0)\ny_pred = np.clip(sub[[\"CE\", \"LAA\"]].values, 1e-15, 1 - 1e-15)\n\n# Weights for each class\nweights = np.array([1, 1])  # Adjust weights as needed\n\n# Calculate log loss\nlog_loss = weighted_log_loss(y_true, y_pred, weights, N)\nprint(\"log_loss:\", log_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T06:43:39.384331Z","iopub.execute_input":"2024-11-18T06:43:39.385176Z","iopub.status.idle":"2024-11-18T06:43:39.4032Z","shell.execute_reply.started":"2024-11-18T06:43:39.385131Z","shell.execute_reply":"2024-11-18T06:43:39.402261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n        \n       \n# 加载预训练的 EfficientNet-B2\nmodel = models.efficientnet_b2(pretrained=True)\n\n# 修改最后的分类层\nnum_classes = 2  # CE和LAA两个类别\nmodel.classifier = torch.nn.Linear(model.classifier[1].in_features, num_classes)\n\n# 设置优化器\noptimizer = optim.Adamax(model.parameters(), lr=0.005)\n\n# 设置学习率调度器\nscheduler = StepLR(optimizer, step_size=150)\n\n# 使用GPU if available\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = model.to(device)\n\nprint(f\"Using EfficientNet-B2 model on {device}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T06:43:44.984908Z","iopub.execute_input":"2024-11-18T06:43:44.985287Z","iopub.status.idle":"2024-11-18T06:43:45.260671Z","shell.execute_reply.started":"2024-11-18T06:43:44.985254Z","shell.execute_reply":"2024-11-18T06:43:45.259608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torchvision import transforms, models\nfrom PIL import Image\nimport pandas as pd\nimport os\nfrom glob import glob\nimport numpy as np\n\n# 1. 加载EfficientNet-B2模型\ndef load_model(model_path=None):\n    model = models.efficientnet_b2(pretrained=True)\n    # 修改最后的分类层\n    model.classifier = torch.nn.Linear(model.classifier[1].in_features, 2)\n    \n    # 如果有预训练权重则加载\n    if model_path and os.path.exists(model_path):\n        model.load_state_dict(torch.load(model_path))\n    \n    return model\n\n# 2. 设置图片路径\nimage_folder = '/kaggle/working/test_images'\nls_imgs_png = glob(os.path.join(image_folder, '*.png'))\n\nprint(f\"找到 {len(ls_imgs_png)} 张图片\")\nif len(ls_imgs_png) > 0:\n    print(\"示例图片路径:\", ls_imgs_png[0])\n\n# 3. 定义图像预处理转换\ntransform = transforms.Compose([\n    transforms.Resize((528, 528)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], \n                        std=[0.229, 0.224, 0.225])\n])\n\n# 4. 预测函数\ndef predict_images(model, image_paths, device):\n    model.eval()\n    predictions = []\n    \n    for img_path in image_paths:\n        img = Image.open(img_path).convert('RGB')\n        img_tensor = transform(img).unsqueeze(0).to(device)\n        \n        with torch.no_grad():\n            outputs = model(img_tensor)\n            probs = torch.nn.functional.softmax(outputs, dim=1)\n            pred = probs.cpu().numpy()[0]\n            predictions.append([pred[1], pred[0]])  # 交换顺序\n    \n    return np.array(predictions)\n\n# 5. 加载模型并进行预测\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = load_model('path_to_your_model_weights.pth')  # 如果有预训练权重,填入路径\nmodel = model.to(device)\npredictions = predict_images(model, ls_imgs_png, device)\n\n# 6. 创建结果列表\nresults = []\nlabels = ['CE', 'LAA']\n\nfor i, pred in enumerate(predictions):\n    result = {\n        'filename': os.path.basename(ls_imgs_png[i]),\n        'CE': pred[0],\n        'LAA': pred[1],\n        'predicted_class': labels[np.argmax(pred)],\n        'confidence': np.max(pred)\n    }\n    results.append(result)\n\n# 7. 创建DataFrame并显示结果\ndf_pred = pd.DataFrame(results)\n\nprint(\"\\n详细预测结果:\")\nfor _, row in df_pred.iterrows():\n    print(f\"图片: {row['filename']}\")\n    print(f\"预测类别: {row['predicted_class']}\")\n    print(f\"置信度: {row['confidence']:.2%}\")\n    print(f\"各类别概率: CE={row['CE']:.2%}, LAA={row['LAA']:.2%}\")\n    print()\n\n# 8. 保存预测结果(可选)\ndf_pred.to_csv('predictions.csv', index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T06:43:54.958152Z","iopub.execute_input":"2024-11-18T06:43:54.95855Z","iopub.status.idle":"2024-11-18T06:43:55.470389Z","shell.execute_reply.started":"2024-11-18T06:43:54.958511Z","shell.execute_reply":"2024-11-18T06:43:55.469385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# Sample data for submission DataFrame\ndata = {\n    \"patient_id\": [\"006388\", \"008e5c\", \"00c058\", \"01adc5\"],\n    \"CE\": [0.5620, 0.5536, 0.5696, 0.5719],\n    \"LAA\": [0.4380, 0.4464, 0.4303, 0.4281]\n}\nsub = pd.DataFrame(data)\n\n# Apply prediction logic\nsub[\"prediction\"] = sub[\"CE\"].apply(lambda x: \"CE\" if x >= 0.5 else \"LAA\")\nsub[\"actual\"] = [\"CE\", \"CE\", \"LAA\", \"LAA\"]\nprint(sub)\n\n# Weighted log loss function\ndef weighted_log_loss(y_true, y_pred, weights, N):\n    y_true = np.array(y_true)\n    y_pred = np.array(y_pred)\n    weights = np.array(weights)\n    N = np.array(N)\n\n    # Compute log loss for each class\n    log_loss_per_class = -np.sum(y_true / N[:, np.newaxis] * np.log(y_pred), axis=0)\n\n    # Weighted log loss\n    weighted_log_loss_value = np.sum(weights * log_loss_per_class) / np.sum(weights)\n\n    return weighted_log_loss_value\n\n# True labels (one-hot encoded)\ny_true = np.array([[1, 0],  # Patient 1: CE\n                   [1, 0],  # Patient 2: CE\n                   [0, 1],  # Patient 3: LAA\n                   [0, 1]]) # Patient 4: LAA\n\n# Sample N values\nN = [545, 545, 205, 205]\n\n# Clip y_pred to avoid log(0)\ny_pred = np.clip(sub[[\"CE\", \"LAA\"]].values, 1e-15, 1 - 1e-15)\n\n# Weights for each class\nweights = np.array([1, 1])  # Adjust weights as needed\n\n# Calculate log loss\nlog_loss = weighted_log_loss(y_true, y_pred, weights, N)\nprint(\"log_loss:\", log_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-18T12:21:22.91978Z","iopub.execute_input":"2024-11-18T12:21:22.920563Z","iopub.status.idle":"2024-11-18T12:21:23.303452Z","shell.execute_reply.started":"2024-11-18T12:21:22.920517Z","shell.execute_reply":"2024-11-18T12:21:23.302513Z"}},"outputs":[],"execution_count":null}]}