{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112899,"databundleVersionId":13449579,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nfrom __future__ import annotations\nfrom pathlib import Path\nfrom typing import List, Tuple\nimport os\nimport numpy as np\nfrom numpy.lib.format import open_memmap\nimport pandas as pd\nimport cv2\nfrom concurrent.futures import ProcessPoolExecutor, as_completed\nfrom tqdm import tqdm\n\n# =====================\n# CONFIG (ajuste aqui)\n# =====================\nTRAIN_CSV = \"/kaggle/input/grand-xray-slam-division-a/train1.csv\"\nTEST_CSV  = \"/kaggle/input/grand-xray-slam-division-a/sample_submission_1.csv\"   # precisa ter 'Image_name'\nTRAIN_DIR = \"/kaggle/input/grand-xray-slam-division-a/train1\"\nTEST_DIR  = \"/kaggle/input/grand-xray-slam-division-a/test1\"\n\nOUT_DIR    = \"/kaggle/working/\"\nIMAGE_SIZE = 320\nCHANNELS   = 1           # salvar compacto (grayscale). No treino repetimos p/ 3 canais.\nDTYPE      = \"uint8\"     # \"uint8\" (recomendado) ou \"float16\"\nOVERWRITE  = True\n# Ajuste conforme o seu SSD/CPU. 8–16 costuma ser bom em NVMe. Em HDD, reduza.\nN_WORKERS  = max(1, (os.cpu_count() or 2) - 1)\n\nLABEL_COLUMNS = [\n    'Atelectasis','Cardiomegaly','Consolidation','Edema','Enlarged Cardiomediastinum',\n    'Fracture','Lung Lesion','Lung Opacity','No Finding','Pleural Effusion',\n    'Pleural Other','Pneumonia','Pneumothorax','Support Devices'\n]\n\n# =====================\n# Helpers\n# =====================\ndef _ensure_outdir(p: Path):\n    p.mkdir(parents=True, exist_ok=True)\n\ndef _validate_files(file_paths: List[Path]):\n    missing = [str(p) for p in file_paths if not p.is_file()]\n    if missing:\n        raise FileNotFoundError(f\"{len(missing)} imagens não encontradas. Ex.: {missing[:3]}\")\n\ndef _compute_slices(n: int, n_workers: int, chunk_size: int | None = None) -> List[Tuple[int,int]]:\n    \"\"\"\n    Divide [0, n) em fatias. Se chunk_size for fornecido, cria blocos com esse tamanho;\n    caso contrário, divide em n_workers blocos aproximadamente iguais.\n    Blocos menores -> barra de progresso mais fluida.\n    \"\"\"\n    if chunk_size and chunk_size > 0:\n        slices = []\n        for s in range(0, n, chunk_size):\n            e = min(n, s + chunk_size)\n            if s < e:\n                slices.append((s, e))\n        return slices\n    # fallback: dividir por workers\n    n_workers = max(1, min(n_workers, n))\n    base = n // n_workers\n    rem  = n % n_workers\n    slices = []\n    start = 0\n    for w in range(n_workers):\n        size = base + (1 if w < rem else 0)\n        end = start + size\n        if start < end:\n            slices.append((start, end))\n        start = end\n    return slices\n\ndef _worker_write_slice(\n    out_imgs: str,\n    files_chunk: List[str],\n    idx_start: int,\n    image_size: int,\n):\n    # Reabrir memmap neste processo\n    arr = open_memmap(out_imgs, mode='r+')\n    H = W = image_size\n\n    # Array scratch para evitar realocações\n    scratch = np.empty((H, W), dtype=np.uint8)\n\n    for k, path_str in enumerate(files_chunk):\n        i = idx_start + k\n        # Leitura grayscale rápida\n        img = cv2.imread(path_str, cv2.IMREAD_GRAYSCALE)\n        if img is None:\n            raise RuntimeError(f\"Falha ao ler: {path_str}\")\n\n        # Resize (INTER_AREA é adequado para downscale)\n        if img.shape[0] != H or img.shape[1] != W:\n            img = cv2.resize(img, (W, H), interpolation=cv2.INTER_AREA)\n\n        # Garantir dtype\n        if img.dtype != np.uint8:\n            img = img.astype(np.uint8, copy=False)\n\n        scratch[...] = img\n        # Escrever (C,H,W) com C=1\n        arr[i, 0, :, :] = scratch\n\n    # Fechar view do memmap neste processo\n    del arr\n    return True\n\ndef _pack_split(csv_path: str, img_dir: str, is_train: bool):\n    csv_path = Path(csv_path); img_dir = Path(img_dir)\n    out_dir  = Path(OUT_DIR);  _ensure_outdir(out_dir)\n\n    df = pd.read_csv(csv_path)\n    if 'Image_name' not in df.columns:\n        raise KeyError(\"CSV precisa conter a coluna 'Image_name'.\")\n\n    n = len(df)\n    files: List[Path] = [img_dir / str(nm) for nm in df['Image_name'].tolist()]\n    _validate_files(files)\n\n    c, h, w = CHANNELS, IMAGE_SIZE, IMAGE_SIZE\n\n    if is_train:\n        out_imgs = out_dir / f\"images_train_{IMAGE_SIZE}_c1_uint8.npy\"\n        out_lbls = out_dir / \"labels_train.npy\"\n        out_idx  = out_dir / \"index_train.csv\"\n    else:\n        out_imgs = out_dir / f\"images_test_{IMAGE_SIZE}_c1_uint8.npy\"\n        out_lbls = None\n        out_idx  = out_dir / \"index_test.csv\"\n\n    if out_imgs.exists() and not OVERWRITE:\n        raise FileExistsError(f\"{out_imgs} já existe. Defina OVERWRITE=True para sobrescrever.\")\n\n    print(f\"=== PACK {'TRAIN' if is_train else 'TEST'} ===\")\n    print(f\"N={n} | {h}x{w} | C={c} | dtype=uint8 -> {out_imgs}\")\n\n    # Criar arquivo .npy memmap e pré-alocar\n    arr = open_memmap(str(out_imgs), mode='w+', dtype=np.uint8, shape=(n, c, h, w))\n    del arr  # será reaberto pelos workers em 'r+'\n\n    # Estratégia de progresso:\n    # - blocos menores deixam a barra mais suave; 512/1024 são bons valores.\n    # - ajuste chunk_size se quiser barras mais granulares.\n    chunk_size = 1024\n    slices = _compute_slices(n, N_WORKERS, chunk_size=chunk_size)\n    paths_str = [str(p) for p in files]\n\n    total_done = 0\n    with ProcessPoolExecutor(max_workers=min(N_WORKERS, len(slices))) as ex, \\\n         tqdm(total=n, desc=\"Empacotando (paralelo)\", unit=\"img\", dynamic_ncols=True) as pbar:\n        futures = []\n        for (s, e) in slices:\n            fut = ex.submit(\n                _worker_write_slice,\n                str(out_imgs),\n                paths_str[s:e],\n                s,\n                IMAGE_SIZE,\n            )\n            futures.append((fut, e - s))\n\n        # Conforme cada slice concluir, atualizamos o progresso\n        for fut, size in futures:\n            fut.result()  # Propaga erro, se houver\n            total_done += size\n            pbar.update(size)\n\n    # CSV rápido\n    if is_train:\n        # Checar colunas\n        missing = [c for c in LABEL_COLUMNS if c not in df.columns]\n        if missing:\n            raise KeyError(f\"Colunas faltantes no CSV de treino: {missing}\")\n\n        out_df = pd.DataFrame({\n            \"idx\": np.arange(n, dtype=np.int32),\n            \"path\": paths_str,  # opcional: salvar apenas Image_name\n        })\n        for col in LABEL_COLUMNS:\n            out_df[col] = df[col].astype(np.float32).values\n        out_df.to_csv(out_idx, index=False)\n\n        # Labels separados (compatível com treino)\n        labels = df[LABEL_COLUMNS].astype(np.float32).values\n        np.save(out_lbls, labels)\n        print(f\"Labels salvos em: {out_lbls}\")\n\n    else:\n        out_df = pd.DataFrame({\n            \"idx\": np.arange(n, dtype=np.int32),\n            \"path\": paths_str,\n        })\n        out_df.to_csv(out_idx, index=False)\n\n    print(f\"Imagens salvas em: {out_imgs}\")\n    print(f\"Índice salvo em:  {out_idx}\")\n\n# =====================\n# Main\n# =====================\nif __name__ == \"__main__\":\n    _pack_split(TRAIN_CSV, TRAIN_DIR, is_train=True)\n    _pack_split(TEST_CSV,  TEST_DIR,  is_train=False)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-22T02:15:47.460497Z","iopub.execute_input":"2025-09-22T02:15:47.460998Z","execution_failed":"2025-09-22T02:27:26.231Z"}},"outputs":[],"execution_count":null}]}