{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.16","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594},{"sourceType":"datasetVersion","sourceId":1799839,"datasetId":1069682,"databundleVersionId":1837296},{"sourceType":"datasetVersion","sourceId":1800066,"datasetId":1069809,"databundleVersionId":1837523},{"sourceType":"datasetVersion","sourceId":1800067,"datasetId":1069810,"databundleVersionId":1837524},{"sourceType":"datasetVersion","sourceId":1800825,"datasetId":1070245,"databundleVersionId":1838284},{"sourceType":"datasetVersion","sourceId":1800777,"datasetId":1069787,"databundleVersionId":1838236},{"sourceType":"datasetVersion","sourceId":1800778,"datasetId":1070222,"databundleVersionId":1838237}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Installa le versioni specifiche dei pacchetti\n!pip install numpy\n!pip install torch torchvision torchaudio\n!pip install ultralytics\n\n# Riavvia il runtime dopo l'installazione\n# import os\n# os.kill(os.getpid(), 9)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:30:27.596923Z","iopub.execute_input":"2024-12-06T13:30:27.597757Z","iopub.status.idle":"2024-12-06T13:30:54.324707Z","shell.execute_reply.started":"2024-12-06T13:30:27.597715Z","shell.execute_reply":"2024-12-06T13:30:54.323764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Carica il CSV\ntrain_df = pd.read_csv('/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv')\n\n# Analisi delle duplicazioni\nprint(\"\\nAnalisi delle annotazioni:\")\nprint(\"-\" * 50)\n\n# 1. Conta quante annotazioni ci sono per ogni combinazione immagine-classe\nduplicates = train_df.groupby(['image_id', 'class_name']).size().reset_index(name='count')\nduplicates_filtered = duplicates[duplicates['count'] > 1]\n\nprint(f\"Numero di combinazioni immagine-classe con annotazioni multiple: {len(duplicates_filtered)}\")\n\n# 2. Mostra alcuni esempi\nif len(duplicates_filtered) > 0:\n    print(\"\\nEsempi di annotazioni multiple:\")\n    print(duplicates_filtered.head())\n    \n    # 3. Mostra un esempio dettagliato di un'immagine con annotazioni multiple\n    example_image = duplicates_filtered.iloc[0]['image_id']\n    example_class = duplicates_filtered.iloc[0]['class_name']\n    print(f\"\\nDettaglio annotazioni per immagine {example_image}, classe {example_class}:\")\n    print(train_df[(train_df['image_id'] == example_image) & \n                  (train_df['class_name'] == example_class)])\n\n# 4. Statistiche generali\nprint(\"\\nStatistiche generali:\")\nprint(f\"Numero totale di immagini: {train_df['image_id'].nunique()}\")\nprint(f\"Numero totale di annotazioni: {len(train_df)}\")\nprint(\"\\nMedia annotazioni per immagine per classe:\")\nprint(duplicates['count'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:31:09.265823Z","iopub.execute_input":"2024-12-06T13:31:09.266694Z","iopub.status.idle":"2024-12-06T13:31:09.54396Z","shell.execute_reply.started":"2024-12-06T13:31:09.266656Z","shell.execute_reply":"2024-12-06T13:31:09.543119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n# Carica il CSV\ntrain_df = pd.read_csv('/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv')\n\n# Conta il numero di istanze (annotazioni) per ogni classe\ninstances_per_class = train_df['class_name'].value_counts()\n\n# Conta il numero di immagini uniche per ogni classe\nimages_per_class = train_df.groupby('class_name')['image_id'].nunique().sort_values(ascending=False)\n\nprint(\"Statistiche per classe:\")\nprint(\"-\" * 70)\nprint(f\"{'Classe':20} {'Immagini':8} {'Istanze':8} {'Media istanze/immagine':20}\")\nprint(\"-\" * 70)\n\nfor class_name in images_per_class.index:\n    images = images_per_class[class_name]\n    instances = instances_per_class[class_name]\n    avg_instances = instances / images\n    print(f\"{class_name:20} {images:8d} {instances:8d} {avg_instances:20.2f}\")\n\n# Calcola statistiche generali\ntotal_images = train_df['image_id'].nunique()\ntotal_instances = len(train_df)\n\nprint(\"\\nStatistiche generali:\")\nprint(\"-\" * 50)\nprint(f\"Totale immagini nel dataset: {total_images}\")\nprint(f\"Totale istanze/annotazioni: {total_instances}\")\nprint(f\"Media istanze per immagine: {total_instances/total_images:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:31:14.321474Z","iopub.execute_input":"2024-12-06T13:31:14.322142Z","iopub.status.idle":"2024-12-06T13:31:14.441532Z","shell.execute_reply.started":"2024-12-06T13:31:14.3221Z","shell.execute_reply":"2024-12-06T13:31:14.440615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Carica il CSV\ntrain_df = pd.read_csv('/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv')\n\n# Conta immagini con \"No finding\" vs immagini con patologie\nnormal_images = train_df[train_df['class_name'] == 'No finding']['image_id'].nunique()\nabnormal_images = train_df[train_df['class_name'] != 'No finding']['image_id'].nunique()\n\nprint(f\"Analisi delle Immagini:\")\nprint(f\"Totale immagini uniche: {train_df['image_id'].nunique()}\")\nprint(f\"Immagini con 'No finding': {normal_images}\")\nprint(f\"Immagini con patologie: {abnormal_images}\")\n\n# Analisi multi-label\nprint(\"\\nAnalisi Multi-label:\")\nimage_labels = train_df.groupby('image_id')['class_name'].nunique()\nprint(f\"Media etichette per immagine: {image_labels.mean():.2f}\")\nprint(f\"Max etichette per immagine: {image_labels.max()}\")\n\n# Distribuzione delle annotazioni per radiologo\nprint(\"\\nAnnotazioni per radiologo:\")\nprint(train_df['rad_id'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:31:17.385435Z","iopub.execute_input":"2024-12-06T13:31:17.386009Z","iopub.status.idle":"2024-12-06T13:31:17.536027Z","shell.execute_reply.started":"2024-12-06T13:31:17.385975Z","shell.execute_reply":"2024-12-06T13:31:17.53513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Carica il CSV\ntrain_df = pd.read_csv('/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv')\n\n# Crea un DataFrame che mostra per ogni immagine quali classi sono state annotate da quali radiologi\ndef analyze_radiologist_agreement():\n    # Raggruppa per immagine e controlla se ci sono diagnosi diverse\n    disagreements = []\n    \n    for image_id in train_df['image_id'].unique():\n        image_annotations = train_df[train_df['image_id'] == image_id]\n        \n        # Ottieni le classi uniche per questa immagine\n        unique_classes = image_annotations['class_name'].unique()\n        \n        if len(unique_classes) > 1:  # Se c'è più di una classe\n            radiologists = {}\n            for _, row in image_annotations.iterrows():\n                rad_id = row['rad_id']\n                class_name = row['class_name']\n                if rad_id not in radiologists:\n                    radiologists[rad_id] = []\n                radiologists[rad_id].append(class_name)\n            \n            # Controlla se i radiologi sono in disaccordo\n            classes_per_rad = set(tuple(sorted(classes)) for classes in radiologists.values())\n            if len(classes_per_rad) > 1:\n                disagreements.append({\n                    'image_id': image_id,\n                    'diagnoses': radiologists\n                })\n    \n    return disagreements\n\ndisagreements = analyze_radiologist_agreement()\n\nprint(f\"Numero totale di immagini con disaccordo tra radiologi: {len(disagreements)}\")\nprint(\"\\nEsempi di disaccordi:\")\nprint(\"-\" * 70)\n\n# Mostra i primi 5 esempi di disaccordo\nfor i, case in enumerate(disagreements[:5]):\n    print(f\"\\nImmagine {i+1}: {case['image_id']}\")\n    for rad_id, diagnoses in case['diagnoses'].items():\n        print(f\"Radiologo {rad_id}: {', '.join(diagnoses)}\")\n\n# Calcola statistiche sul disaccordo\ntotal_images = len(train_df['image_id'].unique())\ndisagreement_rate = (len(disagreements) / total_images) * 100\n\nprint(f\"\\nPercentuale di immagini con disaccordo: {disagreement_rate:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:31:22.897801Z","iopub.execute_input":"2024-12-06T13:31:22.898189Z","iopub.status.idle":"2024-12-06T13:32:41.672366Z","shell.execute_reply.started":"2024-12-06T13:31:22.898159Z","shell.execute_reply":"2024-12-06T13:32:41.671391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport shutil\nimport yaml\nfrom sklearn.model_selection import train_test_split\nfrom ultralytics import YOLO\n\nfrom datetime import datetime\nimport time","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:32:53.948139Z","iopub.execute_input":"2024-12-06T13:32:53.948455Z","iopub.status.idle":"2024-12-06T13:32:57.56316Z","shell.execute_reply.started":"2024-12-06T13:32:53.94843Z","shell.execute_reply":"2024-12-06T13:32:57.562421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport shutil\nimport yaml\nimport time\nfrom datetime import datetime\nfrom sklearn.model_selection import GroupKFold\n\nclass YOLODatasetPreparer:\n    def __init__(self, train_csv, image_dir, output_dir, fold=0, n_splits=5):\n        print(f\"\\n[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Inizializzazione dataset preparer...\")\n        \n        self.train_csv = train_csv\n        self.image_dir = image_dir\n        self.output_dir = output_dir\n        self.fold = fold\n        self.n_splits = n_splits\n        \n        # Crea directory necessarie\n        os.makedirs(os.path.join(output_dir, 'images', 'train'), exist_ok=True)\n        os.makedirs(os.path.join(output_dir, 'images', 'val'), exist_ok=True)\n        os.makedirs(os.path.join(output_dir, 'labels', 'train'), exist_ok=True)\n        os.makedirs(os.path.join(output_dir, 'labels', 'val'), exist_ok=True)\n        \n        # Carica e pre-processa le annotazioni\n        print(f\"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Caricamento e pre-processing dei dati...\")\n        self.train_annotations = pd.read_csv(train_csv)\n        \n        # Normalizza le coordinate usando width e height\n        self.train_annotations['x_min'] = self.train_annotations.apply(lambda row: row.x_min/row.width if pd.notna(row.x_min) else row.x_min, axis=1)\n        self.train_annotations['y_min'] = self.train_annotations.apply(lambda row: row.y_min/row.height if pd.notna(row.y_min) else row.y_min, axis=1)\n        self.train_annotations['x_max'] = self.train_annotations.apply(lambda row: row.x_max/row.width if pd.notna(row.x_max) else row.x_max, axis=1)\n        self.train_annotations['y_max'] = self.train_annotations.apply(lambda row: row.y_max/row.height if pd.notna(row.y_max) else row.y_max, axis=1)\n        \n        # Gestione \"No finding\"\n        no_finding_images = self.train_annotations[\n            self.train_annotations['class_name'] == 'No finding'\n        ]['image_id'].unique()\n        \n        print(f\"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Processamento immagini 'No finding'...\")\n        no_finding_annotations = []\n        for img_id in no_finding_images:\n            if len(self.train_annotations[\n                (self.train_annotations['image_id'] == img_id) & \n                (self.train_annotations['class_name'] != 'No finding')\n            ]) == 0:\n                no_finding_annotations.append({\n                    'image_id': img_id,\n                    'class_name': 'No finding',\n                    'x_min': 0,\n                    'y_min': 0,\n                    'x_max': 1,\n                    'y_max': 1,\n                    'width': 1024,\n                    'height': 1024\n                })\n        \n        if no_finding_annotations:\n            self.train_annotations = pd.concat([\n                self.train_annotations,\n                pd.DataFrame(no_finding_annotations)\n            ], ignore_index=True)\n        \n        # Crea mapping delle classi\n        self.class_map = self.create_class_mapping()\n        \n        # Crea fold\n        self.create_folds()\n        \n        print(f\"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Inizializzazione completata\")\n\n    def create_class_mapping(self):\n        \"\"\"Crea un dizionario di mapping tra nomi delle classi e ID\"\"\"\n        unique_classes = sorted(self.train_annotations['class_name'].unique())\n        class_map = {cls: idx for idx, cls in enumerate(unique_classes)}\n        print(\"Class mapping:\", class_map)\n        return class_map\n    \n    def create_folds(self):\n        \"\"\"Crea i fold usando GroupKFold\"\"\"\n        gkf = GroupKFold(n_splits=self.n_splits)\n        self.train_annotations['fold'] = -1\n        \n        # Assegna i fold basandosi sugli image_id unici\n        unique_images = self.train_annotations[['image_id']].drop_duplicates()\n        for fold_idx, (_, val_idx) in enumerate(gkf.split(\n            unique_images, groups=unique_images.image_id.tolist()\n        )):\n            self.train_annotations.loc[\n                self.train_annotations.image_id.isin(\n                    unique_images.iloc[val_idx].image_id\n                ),\n                'fold'\n            ] = fold_idx\n\n    def create_yolo_annotation(self, image_id):\n        \"\"\"Crea annotazioni nel formato YOLO\"\"\"\n        image_annotations = self.train_annotations[self.train_annotations['image_id'] == image_id]\n        \n        yolo_annotations = []\n        has_no_finding = False\n        \n        for _, row in image_annotations.iterrows():\n            try:\n                class_name = row['class_name']\n                \n                if class_name == 'No finding':\n                    if not has_no_finding:\n                        has_no_finding = True\n                        class_id = self.class_map[class_name]\n                        yolo_annotations.append(f\"{class_id} 0.5 0.5 0.9 0.9\")\n                    continue\n                \n                if pd.isna(row['x_min']) or pd.isna(row['y_min']) or \\\n                   pd.isna(row['x_max']) or pd.isna(row['y_max']):\n                    continue\n                \n                # Calcola centro e dimensioni (le coordinate sono già normalizzate)\n                x_center = (row['x_min'] + row['x_max']) / 2\n                y_center = (row['y_min'] + row['y_max']) / 2\n                w = row['x_max'] - row['x_min']\n                h = row['y_max'] - row['y_min']\n                \n                if 0 <= x_center <= 1 and 0 <= y_center <= 1 and w > 0 and h > 0:\n                    class_id = self.class_map[class_name]\n                    annotation = f\"{class_id} {x_center:.6f} {y_center:.6f} {w:.6f} {h:.6f}\"\n                    yolo_annotations.append(annotation)\n            \n            except Exception as e:\n                print(f\"Error processing {class_name} in {image_id}: {str(e)}\")\n                continue\n        \n        return yolo_annotations\n\n    def _process_split(self, image_ids, split):\n        \"\"\"Processa immagini e label per train o validation\"\"\"\n        processed = 0\n        start_time = time.time()\n        \n        for image_id in image_ids:\n            # Percorsi dei file\n            png_path = os.path.join(self.image_dir, f\"{image_id}.png\")\n            label_path = os.path.join(\n                self.output_dir, 'labels', split, f\"{image_id}.txt\"\n            )\n            dest_image_path = os.path.join(\n                self.output_dir, 'images', split, f\"{image_id}.png\"\n            )\n            \n            try:\n                # Copia immagine\n                shutil.copy(png_path, dest_image_path)\n                \n                # Crea e salva annotazioni YOLO\n                yolo_annotations = self.create_yolo_annotation(image_id)\n                if yolo_annotations:\n                    with open(label_path, 'w') as f:\n                        f.write('\\n'.join(yolo_annotations))\n                \n                processed += 1\n                if processed % 1000 == 0:\n                    elapsed_time = time.time() - start_time\n                    speed = processed / elapsed_time\n                    print(f\"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] \"\n                          f\"Processed {processed}/{len(image_ids)} images for {split} \"\n                          f\"({speed:.1f} img/s)\")\n                \n            except Exception as e:\n                print(f\"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] \"\n                      f\"Error processing {image_id}: {str(e)}\")\n                continue\n\n    def prepare_yolo_dataset(self):\n        \"\"\"Prepara il dataset YOLO usando il fold specificato\"\"\"\n        # Pulisci le directory di output\n        train_dir = os.path.join(self.output_dir, 'images', 'train')\n        val_dir = os.path.join(self.output_dir, 'images', 'val')\n        train_labels = os.path.join(self.output_dir, 'labels', 'train')\n        val_labels = os.path.join(self.output_dir, 'labels', 'val')\n        \n        for dir_path in [train_dir, val_dir, train_labels, val_labels]:\n            if os.path.exists(dir_path):\n                shutil.rmtree(dir_path)\n            os.makedirs(dir_path)\n        \n        # Split basato sui fold\n        train_ids = self.train_annotations[\n            self.train_annotations['fold'] != self.fold\n        ]['image_id'].unique()\n        val_ids = self.train_annotations[\n            self.train_annotations['fold'] == self.fold\n        ]['image_id'].unique()\n        \n        print(f\"\\n[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Processing fold {self.fold}\")\n        print(f\"Training images: {len(train_ids)}\")\n        print(f\"Validation images: {len(val_ids)}\")\n        \n        # Processa gli split\n        self._process_split(train_ids, 'train')\n        self._process_split(val_ids, 'val')\n        \n        # Crea file YAML\n        yaml_config = {\n            'path': self.output_dir,\n            'train': 'images/train',\n            'val': 'images/val',\n            'nc': len(self.class_map),\n            'names': list(self.class_map.keys())\n        }\n        \n        yaml_path = os.path.join(self.output_dir, 'dataset.yaml')\n        with open(yaml_path, 'w') as f:\n            yaml.dump(yaml_config, f, default_flow_style=False)\n        \n        print(f\"\\nConfig saved to {yaml_path}\")\n        return yaml_path","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:33:03.092126Z","iopub.execute_input":"2024-12-06T13:33:03.092587Z","iopub.status.idle":"2024-12-06T13:33:03.118435Z","shell.execute_reply.started":"2024-12-06T13:33:03.092558Z","shell.execute_reply":"2024-12-06T13:33:03.117522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preparer = YOLODatasetPreparer(\n    train_csv='/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv',\n    image_dir='/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train',\n    output_dir='/kaggle/working/yolo_dataset'\n)\n\nyaml_path = preparer.prepare_yolo_dataset()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:33:10.786303Z","iopub.execute_input":"2024-12-06T13:33:10.786638Z","iopub.status.idle":"2024-12-06T13:40:52.97552Z","shell.execute_reply.started":"2024-12-06T13:33:10.786609Z","shell.execute_reply":"2024-12-06T13:40:52.974636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import yaml\n\nyaml_path = '/kaggle/working/yolo_dataset/dataset.yaml'\n# Verifica il contenuto del file YAML\nwith open(yaml_path, 'r') as f:\n    config = yaml.safe_load(f)\n    \n# Verifica le immagini\nval_path = os.path.join(config['path'], config['val'])\ntrain_path = os.path.join(config['path'], config['train'])\n\nprint(f\"Immagini nel training set: {len(os.listdir(train_path))}\")\nprint(f\"Immagini nel validation set: {len(os.listdir(val_path))}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:41:57.940712Z","iopub.execute_input":"2024-12-06T13:41:57.941514Z","iopub.status.idle":"2024-12-06T13:41:57.957765Z","shell.execute_reply.started":"2024-12-06T13:41:57.941479Z","shell.execute_reply":"2024-12-06T13:41:57.956878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Addestramento\nimport wandb\n#wandb.login(key='d0d96c9a2f07c81db580937bffde63589ab42fba')\n#wandb.init(mode=\"offline\")\nwandb.init(mode=\"disabled\")\n\n# model = YOLO('/kaggle/working/runs/train/yolov8n_improved/weights/best.pt')\nmodel = YOLO('yolo11n.pt')\nresults = model.train(\n    data=yaml_path,\n    epochs=5,                    \n    imgsz=1024,                  \n    batch=16,                   \n    device=0,\n    cache=False,                  \n    patience=20,               \n    plots=True,\n    save_period=5,              \n    workers=6,                  \n    seed=42,\n    resume=False,\n    project='/kaggle/working/runs/train',\n    name='yolov11n_improved',    \n    exist_ok=True,            \n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:45:32.326068Z","iopub.execute_input":"2024-12-06T13:45:32.326851Z","iopub.status.idle":"2024-12-06T14:17:49.087696Z","shell.execute_reply.started":"2024-12-06T13:45:32.326816Z","shell.execute_reply":"2024-12-06T14:17:49.086796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Carica il CSV\ntrain_df = pd.read_csv('/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv')\n\n# Conta immagini con \"No finding\" vs immagini con patologie\nnormal_images = train_df[train_df['class_name'] == 'No finding']['image_id'].nunique()\nabnormal_images = train_df[train_df['class_name'] != 'No finding']['image_id'].nunique()\n\nprint(f\"Analisi delle Immagini:\")\nprint(f\"Totale immagini uniche: {train_df['image_id'].nunique()}\")\nprint(f\"Immagini con 'No finding': {normal_images}\")\nprint(f\"Immagini con patologie: {abnormal_images}\")\n\n# Analisi multi-label\nprint(\"\\nAnalisi Multi-label:\")\nimage_labels = train_df.groupby('image_id')['class_name'].nunique()\nprint(f\"Media etichette per immagine: {image_labels.mean():.2f}\")\nprint(f\"Max etichette per immagine: {image_labels.max()}\")\n\n# Distribuzione delle annotazioni per radiologo\nprint(\"\\nAnnotazioni per radiologo:\")\nprint(train_df['rad_id'].value_counts())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport yaml\n\n# Definizione del contenuto del file di configurazione\nconfig = {\n    'path': '/kaggle/working/yolo_dataset',\n    'train': 'images/train',\n    'val': 'images/val',\n    'names': {\n        0: 'No finding',\n        1: 'Aortic enlargement',\n        2: 'Cardiomegaly',\n        3: 'Pleural thickening',\n        4: 'Pulmonary fibrosis',\n        5: 'Lung Opacity',\n        6: 'Other lesion',\n        7: 'Pleural effusion',\n        8: 'Nodule/Mass',\n        9: 'Infiltration',\n        10: 'Calcification',\n        11: 'ILD',\n        12: 'Consolidation',\n        13: 'Atelectasis',\n        14: 'Pneumothorax'\n    },\n    'nc': 15,\n    'class_weights': {\n        0: 1.0,    # No finding - classe base\n        1: 4.4,    # Aortic enlargement\n        2: 5.9,    # Cardiomegaly\n        3: 6.6,    # Pleural thickening\n        4: 6.8,    # Pulmonary fibrosis\n        5: 16.0,   # Lung Opacity\n        6: 18.1,   # Other lesion\n        7: 16.1,   # Pleural effusion\n        8: 9.3,    # Nodule/Mass - ridotto per evitare overfitting\n        9: 38.3,   # Infiltration\n        10: 49.7,  # Calcification\n        11: 47.7,  # ILD\n        12: 100.0, # Consolidation\n        13: 228.3, # Atelectasis\n        14: 281.6  # Pneumothorax\n    }\n}\n\n# Crea la directory se non esiste\nos.makedirs('/kaggle/working/yolo_dataset', exist_ok=True)\n\n# Scrivi il file YAML\nconfig_path = '/kaggle/working/yolo_dataset/yolo-config.yaml'\nwith open(config_path, 'w') as f:\n    yaml.safe_dump(config, f, sort_keys=False, default_flow_style=False)\n\nprint(f\"File di configurazione creato in: {config_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T08:40:22.300494Z","iopub.execute_input":"2024-12-06T08:40:22.300805Z","iopub.status.idle":"2024-12-06T08:40:22.312742Z","shell.execute_reply.started":"2024-12-06T08:40:22.300774Z","shell.execute_reply":"2024-12-06T08:40:22.311975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import wandb\nmodel = YOLO('yolov8n.pt')\nwandb.init(mode=\"disabled\")\n\n# Training configuration con focus su classi sbilanciate\nmodel.train(\n    # data='/kaggle/working/yolo_dataset/yolo-config.yaml',\n    data='/kaggle/working/yolo_dataset/dataset.yaml',\n    save_period=5,        \n    device=0,\n    workers=2,\n    epochs=1,         \n    imgsz=1024,\n    batch=16,           \n    patience=20,      \n    box=7.5,            \n    cls=0.5,          \n    dfl=1.5,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T12:30:31.341121Z","iopub.execute_input":"2024-12-06T12:30:31.341441Z","iopub.status.idle":"2024-12-06T12:47:50.600034Z","shell.execute_reply.started":"2024-12-06T12:30:31.341416Z","shell.execute_reply":"2024-12-06T12:47:50.599269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nfrom pathlib import Path\nimport random\nimport matplotlib.pyplot as plt\nfrom collections import defaultdict\n\ndef validate_yolo_dataset(dataset_path, samples_per_class=1):\n    \"\"\"\n    Valida il dataset YOLO visualizzando almeno un'immagine per ogni classe.\n    \n    Args:\n        dataset_path: percorso alla cartella del dataset\n        samples_per_class: numero minimo di immagini da visualizzare per classe\n    \"\"\"\n    # Classi nel corretto ordine\n    classes = {\n        0: 'Aortic enlargement',\n        1: 'Atelectasis',\n        2: 'Calcification',\n        3: 'Cardiomegaly',\n        4: 'Consolidation',\n        5: 'ILD',\n        6: 'Infiltration',\n        7: 'Lung Opacity',\n        8: 'No finding',\n        9: 'Nodule/Mass',\n        10: 'Other lesion',\n        11: 'Pleural effusion',\n        12: 'Pleural thickening',\n        13: 'Pneumothorax',\n        14: 'Pulmonary fibrosis'\n    }\n    \n    # Percorsi alle immagini e label\n    images_path = Path(dataset_path) / 'images' / 'train'\n    labels_path = Path(dataset_path) / 'labels' / 'train'\n    \n    # Verifica esistenza directory\n    if not images_path.exists() or not labels_path.exists():\n        raise ValueError(f\"Directory non trovate in {dataset_path}\")\n    \n    # Lista tutte le immagini e relative label\n    image_files = list(images_path.glob('*.jpg')) + list(images_path.glob('*.png'))\n    if not image_files:\n        raise ValueError(f\"Nessuna immagine trovata in {images_path}\")\n    \n    # Dizionario per tenere traccia delle immagini per classe\n    images_by_class = defaultdict(list)\n    \n    # Statistiche\n    stats = {\n        'total_images': len(image_files),\n        'missing_labels': 0,\n        'invalid_boxes': 0,\n        'boxes_per_class': {i: 0 for i in range(len(classes))}\n    }\n    \n    # Prima passata: categorizza le immagini per classe\n    for img_path in image_files:\n        label_path = labels_path / f\"{img_path.stem}.txt\"\n        if not label_path.exists():\n            stats['missing_labels'] += 1\n            continue\n            \n        with open(label_path) as f:\n            for line in f:\n                try:\n                    class_id = int(float(line.strip().split()[0]))\n                    images_by_class[class_id].append(img_path)\n                    stats['boxes_per_class'][class_id] += 1\n                except:\n                    continue\n    \n    # Seleziona immagini da visualizzare\n    images_to_show = []\n    for class_id in classes.keys():\n        if class_id in images_by_class:\n            # Prendi samples_per_class immagini per questa classe\n            selected = random.sample(\n                images_by_class[class_id],\n                min(samples_per_class, len(images_by_class[class_id]))\n            )\n            images_to_show.extend(selected)\n    \n    # Rimuovi duplicati mantenendo l'ordine\n    images_to_show = list(dict.fromkeys(images_to_show))\n    \n    print(f\"\\nMostrando {len(images_to_show)} immagini...\\n\")\n    \n    for img_path in images_to_show:\n        # Carica immagine\n        img = cv2.imread(str(img_path))\n        if img is None:\n            print(f\"Errore nel caricamento dell'immagine: {img_path}\")\n            continue\n            \n        height, width = img.shape[:2]\n        \n        # Trova il corrispondente file label\n        label_path = labels_path / f\"{img_path.stem}.txt\"\n        if not label_path.exists():\n            continue\n            \n        # Leggi le label\n        boxes = []\n        class_ids_in_image = set()\n        with open(label_path) as f:\n            for line in f:\n                try:\n                    class_id, x, y, w, h = map(float, line.strip().split())\n                    class_id = int(class_id)\n                    class_ids_in_image.add(class_id)\n                    \n                    if not (0 <= x <= 1 and 0 <= y <= 1 and 0 <= w <= 1 and 0 <= h <= 1):\n                        stats['invalid_boxes'] += 1\n                        continue\n                        \n                    # Converti da formato YOLO a pixel\n                    x1 = int((x - w/2) * width)\n                    y1 = int((y - h/2) * height)\n                    x2 = int((x + w/2) * width)\n                    y2 = int((y + h/2) * height)\n                    \n                    boxes.append((class_id, (x1, y1, x2, y2)))\n                except:\n                    print(f\"Errore nel parsing della label: {line.strip()} in {label_path}\")\n                    continue\n        \n        # Disegna i box\n        for class_id, (x1, y1, x2, y2) in boxes:\n            color = plt.cm.rainbow(class_id / len(classes))\n            color = tuple(int(255 * c) for c in color[:3])  # Converti in BGR\n            cv2.rectangle(img, (x1, y1), (x2, y2), color, 2)\n            cv2.putText(img, classes[class_id], (x1, y1-10), \n                       cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)\n        \n        # Mostra l'immagine\n        plt.figure(figsize=(15, 10))\n        plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n        plt.title(f\"Image: {img_path.name}\\nClassi presenti: {', '.join(classes[cid] for cid in class_ids_in_image)}\")\n        plt.axis('off')\n        plt.show()\n    \n    # Stampa statistiche\n    print(\"\\nStatistiche Dataset:\")\n    print(f\"Totale immagini: {stats['total_images']}\")\n    print(f\"Label mancanti: {stats['missing_labels']}\")\n    print(f\"Box invalidi: {stats['invalid_boxes']}\")\n    print(\"\\nDistribuzione box per classe:\")\n    for class_id, count in stats['boxes_per_class'].items():\n        print(f\"{classes[class_id]}: {count}\")\n    \n    return stats\n\n# Uso:\ndataset_path = '/kaggle/working/yolo_dataset'\nstats = validate_yolo_dataset(dataset_path, samples_per_class=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:42:09.041019Z","iopub.execute_input":"2024-12-06T13:42:09.041393Z","iopub.status.idle":"2024-12-06T13:42:16.471786Z","shell.execute_reply.started":"2024-12-06T13:42:09.041361Z","shell.execute_reply":"2024-12-06T13:42:16.470938Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom collections import defaultdict\n\ndef validate_box_dimensions(w, h, img_width, img_height, min_size=10, max_ratio=0.9):\n    \"\"\"\n    Verifica che le dimensioni del box siano ragionevoli\n    \"\"\"\n    # Converti in pixel\n    w_px = w * img_width\n    h_px = h * img_height\n    \n    # Controlla dimensione minima\n    if w_px < min_size or h_px < min_size:\n        return False, \"Box troppo piccolo\"\n    \n    # Controlla che il box non sia troppo grande rispetto all'immagine\n    if w > max_ratio or h > max_ratio:\n        return False, \"Box troppo grande rispetto all'immagine\"\n        \n    # Controlla aspect ratio (evita box estremamente stretti o larghi)\n    aspect_ratio = w_px / h_px if h_px > 0 else float('inf')\n    if aspect_ratio > 10 or aspect_ratio < 0.1:\n        return False, f\"Aspect ratio sospetto: {aspect_ratio:.2f}\"\n        \n    return True, \"\"\n\ndef check_yolo_labels(label_path, img_width, img_height):\n    \"\"\"\n    Verifica il formato e la validità delle label YOLO\n    \"\"\"\n    errors = []\n    boxes = []\n    \n    try:\n        with open(label_path) as f:\n            for i, line in enumerate(f.readlines(), 1):\n                parts = line.strip().split()\n                \n                # Verifica formato base\n                if len(parts) != 5:\n                    errors.append(f\"Linea {i}: numero errato di valori (atteso 5, trovato {len(parts)})\")\n                    continue\n                \n                try:\n                    class_id, x_center, y_center, width, height = map(float, parts)\n                    class_id = int(class_id)\n                except ValueError:\n                    errors.append(f\"Linea {i}: errore di conversione dei valori in numeri\")\n                    continue\n                \n                # Verifica range delle coordinate (devono essere tra 0 e 1)\n                if not all(0 <= val <= 1 for val in [x_center, y_center, width, height]):\n                    errors.append(f\"Linea {i}: coordinate fuori range [0,1]\")\n                    continue\n                \n                # Verifica dimensioni ragionevoli\n                valid, message = validate_box_dimensions(width, height, img_width, img_height)\n                if not valid:\n                    errors.append(f\"Linea {i}: {message}\")\n                    continue\n                \n                # Verifica che il box non esca dall'immagine\n                if x_center - width/2 < 0 or x_center + width/2 > 1 or \\\n                   y_center - height/2 < 0 or y_center + height/2 > 1:\n                    errors.append(f\"Linea {i}: box esce dai limiti dell'immagine\")\n                    continue\n                \n                # Se arriviamo qui, il box è valido\n                boxes.append({\n                    'class_id': class_id,\n                    'x_center': x_center,\n                    'y_center': y_center,\n                    'width': width,\n                    'height': height,\n                    'original_line': line.strip()\n                })\n                \n    except Exception as e:\n        errors.append(f\"Errore nella lettura del file: {str(e)}\")\n    \n    return boxes, errors\n\ndef visualize_box_validation(dataset_path, samples_per_class=1):\n    \"\"\"\n    Visualizza e valida i box del dataset\n    \"\"\"\n    classes = {\n        0: 'Aortic enlargement',\n        1: 'Atelectasis',\n        2: 'Calcification',\n        3: 'Cardiomegaly',\n        4: 'Consolidation',\n        5: 'ILD',\n        6: 'Infiltration',\n        7: 'Lung Opacity',\n        8: 'No finding',\n        9: 'Nodule/Mass',\n        10: 'Other lesion',\n        11: 'Pleural effusion',\n        12: 'Pleural thickening',\n        13: 'Pneumothorax',\n        14: 'Pulmonary fibrosis'\n    }\n    \n    images_path = Path(dataset_path) / 'images' / 'train'\n    labels_path = Path(dataset_path) / 'labels' / 'train'\n    \n    if not images_path.exists() or not labels_path.exists():\n        raise ValueError(f\"Directory non trovate in {dataset_path}\")\n    \n    # Raccogli tutte le immagini con i loro box\n    images_with_errors = []\n    total_processed = 0\n    \n    for img_path in sorted(images_path.glob('*.jpg')):\n        total_processed += 1\n        img = cv2.imread(str(img_path))\n        if img is None:\n            print(f\"Errore nel caricamento dell'immagine: {img_path}\")\n            continue\n            \n        height, width = img.shape[:2]\n        label_path = labels_path / f\"{img_path.stem}.txt\"\n        \n        if not label_path.exists():\n            print(f\"Label mancante per: {img_path}\")\n            continue\n            \n        # Valida le label\n        boxes, errors = check_yolo_labels(label_path, width, height)\n        \n        if errors:\n            print(f\"\\nErrori trovati in {img_path.name}:\")\n            for error in errors:\n                print(f\"- {error}\")\n            images_with_errors.append(img_path.name)\n        \n        # Visualizza l'immagine con i box\n        plt.figure(figsize=(15, 10))\n        plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n        \n        # Disegna i box originali in verde\n        for box in boxes:\n            x_center = box['x_center']\n            y_center = box['y_center']\n            w = box['width']\n            h = box['height']\n            \n            # Converti in pixel\n            x1 = int((x_center - w/2) * width)\n            y1 = int((y_center - h/2) * height)\n            x2 = int((x_center + w/2) * width)\n            y2 = int((y_center + h/2) * height)\n            \n            # Disegna box e label\n            plt.gca().add_patch(plt.Rectangle((x1, y1), x2-x1, y2-y1, \n                                            fill=False, color='green', linewidth=2))\n            plt.text(x1, y1-10, classes[box['class_id']], \n                    color='green', fontsize=12, backgroundcolor='white')\n            \n            # Disegna il centro del box per verifica\n            plt.plot(x_center * width, y_center * height, 'r+', markersize=10)\n        \n        plt.title(f\"Image: {img_path.name}\\n\" + \n                 f\"Dimensioni: {width}x{height}\\n\" +\n                 f\"Numero di box: {len(boxes)}\")\n        plt.axis('on')  # Mostra gli assi per verificare le coordinate\n        plt.grid(True)  # Aggiungi griglia per riferimento\n        plt.show()\n        \n        # Stampa dettagli dei box per verifica\n        print(\"\\nDettagli dei box:\")\n        for box in boxes:\n            print(f\"Classe: {classes[box['class_id']]}\")\n            print(f\"Coordinate normalizzate: x={box['x_center']:.3f}, y={box['y_center']:.3f}, w={box['width']:.3f}, h={box['height']:.3f}\")\n            print(f\"Riga originale: {box['original_line']}\")\n            print(\"---\")\n        \n        if total_processed >= samples_per_class:\n            break\n    \n    print(f\"\\nProcessate {total_processed} immagini\")\n    if images_with_errors:\n        print(f\"\\nImmagini con errori ({len(images_with_errors)}):\")\n        for img_name in images_with_errors:\n            print(f\"- {img_name}\")\n\n# Uso\ndataset_path = '/kaggle/working/yolo_dataset'\nvisualize_box_validation(dataset_path, samples_per_class=3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T10:20:15.227278Z","iopub.execute_input":"2024-12-06T10:20:15.227565Z","iopub.status.idle":"2024-12-06T10:20:15.272774Z","shell.execute_reply.started":"2024-12-06T10:20:15.227542Z","shell.execute_reply":"2024-12-06T10:20:15.272093Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Leggi il CSV originale\ndf = pd.read_csv('/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train.csv')\n\n# Filtra per l'immagine specifica\nimage_id = 'd4e8665286a7fd59bcd408868c247a5d'\nimage_annotations = df[df['image_id'] == image_id]\n\nprint(f\"\\nAnnotazioni per l'immagine {image_id}:\")\nprint(\"\\nDati completi:\")\nprint(image_annotations.to_string())\n\n# Statistiche sulle coordinate\nif not image_annotations.empty:\n    x_coords = image_annotations[['x_min', 'x_max']].values.flatten()\n    y_coords = image_annotations[['y_min', 'y_max']].values.flatten()\n    \n    # Rimuovi i NaN\n    x_coords = x_coords[~np.isnan(x_coords)]\n    y_coords = y_coords[~np.isnan(y_coords)]\n    \n    if len(x_coords) > 0 and len(y_coords) > 0:\n        print(\"\\nStatistiche coordinate:\")\n        print(f\"Range X: {x_coords.min():.1f} - {x_coords.max():.1f}\")\n        print(f\"Range Y: {y_coords.min():.1f} - {y_coords.max():.1f}\")\n        print(f\"Dimensioni area annotata: {x_coords.max() - x_coords.min():.1f} x {y_coords.max() - y_coords.min():.1f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T10:47:18.482562Z","iopub.execute_input":"2024-12-06T10:47:18.482894Z","iopub.status.idle":"2024-12-06T10:47:18.630119Z","shell.execute_reply.started":"2024-12-06T10:47:18.482863Z","shell.execute_reply":"2024-12-06T10:47:18.629417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_id = '96f334aaa1d7d9cad160e710c9965615'\nimage_annotations = df[df['image_id'] == image_id]\nprint(f\"\\nAnnotazioni per l'immagine {image_id}:\")\nprint(image_annotations[['class_name', 'x_min', 'y_min', 'x_max', 'y_max']].to_string())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T11:31:24.15044Z","iopub.execute_input":"2024-12-06T11:31:24.150784Z","iopub.status.idle":"2024-12-06T11:31:24.166201Z","shell.execute_reply.started":"2024-12-06T11:31:24.150756Z","shell.execute_reply":"2024-12-06T11:31:24.165469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\n\ndef show_original_data(image_id, train_csv_path, image_dir_path):\n    \"\"\"\n    Mostra le informazioni originali per una specifica immagine\n    \"\"\"\n    # Leggi il CSV\n    df = pd.read_csv(train_csv_path)\n    \n    # Filtra per l'immagine specifica\n    image_annotations = df[df['image_id'] == image_id]\n    \n    # Mostra le annotazioni\n    print(f\"\\nAnnotazioni per l'immagine {image_id}:\")\n    print(\"\\nDati dal CSV:\")\n    print(image_annotations[['class_name', 'x_min', 'y_min', 'x_max', 'y_max']].to_string())\n    \n    # Leggi l'immagine\n    img_path = f\"{image_dir_path}/{image_id}.png\"\n    img = cv2.imread(img_path)\n    \n    if img is not None:\n        height, width = img.shape[:2]\n        print(f\"\\nDimensioni immagine originale: {width}x{height}\")\n        \n        # Converti da BGR a RGB per matplotlib\n        img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # Mostra l'immagine con i box originali\n        plt.figure(figsize=(15, 15))\n        plt.imshow(img_rgb)\n        \n        # Disegna i box originali\n        colors = plt.cm.rainbow(np.linspace(0, 1, len(image_annotations)))\n        for idx, ann in image_annotations.iterrows():\n            x_min, y_min = int(ann['x_min']), int(ann['y_min'])\n            x_max, y_max = int(ann['x_max']), int(ann['y_max'])\n            \n            # Disegna il rettangolo\n            rect = plt.Rectangle((x_min, y_min), x_max - x_min, y_max - y_min,\n                               fill=False, edgecolor=colors[idx % len(colors)], linewidth=2)\n            plt.gca().add_patch(rect)\n            \n            # Aggiungi label\n            plt.text(x_min, y_min - 5, ann['class_name'], \n                    color=colors[idx % len(colors)], fontsize=10,\n                    bbox=dict(facecolor='white', alpha=0.7))\n        \n        plt.title(f\"Immagine originale: {image_id}\\ncon box originali\")\n        plt.axis('off')\n        plt.show()\n        \n        # Statistiche sui box\n        print(\"\\nStatistiche coordinate:\")\n        x_coords = image_annotations[['x_min', 'x_max']].values.flatten()\n        y_coords = image_annotations[['y_min', 'y_max']].values.flatten()\n        \n        print(f\"Range X: {x_coords.min():.1f} - {x_coords.max():.1f}\")\n        print(f\"Range Y: {y_coords.min():.1f} - {y_coords.max():.1f}\")\n        print(f\"Area annotata: {x_coords.max() - x_coords.min():.1f} x {y_coords.max() - y_coords.min():.1f}\")\n    else:\n        print(f\"Errore nel caricamento dell'immagine: {img_path}\")\n\n# Uso\nimage_id = 'a3411581087b63a70a2bdba24e55ff4e'\nshow_original_data(\n    image_id,\n    '/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv',\n    '/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train'\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T12:03:53.854976Z","iopub.execute_input":"2024-12-06T12:03:53.855294Z","iopub.status.idle":"2024-12-06T12:03:54.504696Z","shell.execute_reply.started":"2024-12-06T12:03:53.855266Z","shell.execute_reply":"2024-12-06T12:03:54.503361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T19:43:56.925202Z","iopub.execute_input":"2025-03-21T19:43:56.92557Z","iopub.status.idle":"2025-03-21T19:43:56.929675Z","shell.execute_reply.started":"2025-03-21T19:43:56.925538Z","shell.execute_reply":"2025-03-21T19:43:56.928574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Define the file path\nfile_path = \"/kaggle/input/vinbigdata-original-image-dataset/vinbigdata/train.csv\"\n\n# Read the CSV file\ndf = pd.read_csv(file_path)\n\n# Display the first few rows\nprint(df.head(5))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T20:27:35.408413Z","iopub.execute_input":"2025-03-21T20:27:35.408849Z","iopub.status.idle":"2025-03-21T20:27:36.01485Z","shell.execute_reply.started":"2025-03-21T20:27:35.408809Z","shell.execute_reply":"2025-03-21T20:27:36.013894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df[[\"image_id\", \"class_name\", \"class_id\"]]\ndf.head()\nprint(df['class_name'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T20:27:39.944357Z","iopub.execute_input":"2025-03-21T20:27:39.944716Z","iopub.status.idle":"2025-03-21T20:27:39.97151Z","shell.execute_reply.started":"2025-03-21T20:27:39.944686Z","shell.execute_reply":"2025-03-21T20:27:39.97058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df.shape)\ndf.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T20:08:00.035806Z","iopub.execute_input":"2025-03-21T20:08:00.036174Z","iopub.status.idle":"2025-03-21T20:08:00.047126Z","shell.execute_reply.started":"2025-03-21T20:08:00.036143Z","shell.execute_reply":"2025-03-21T20:08:00.046059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\n\n# Define the image directory path\nimage_dir = \"/kaggle/input/vinbigdata-original-image-dataset/vinbigdata/train\"\n\n# Count the number of image files (assuming they are in .jpg or .png format)\nimage_count = len(glob.glob(os.path.join(image_dir, \"*.*\")))\n\nprint(f\"Number of images: {image_count}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T19:50:23.989679Z","iopub.execute_input":"2025-03-21T19:50:23.990023Z","iopub.status.idle":"2025-03-21T19:50:25.104838Z","shell.execute_reply.started":"2025-03-21T19:50:23.989994Z","shell.execute_reply":"2025-03-21T19:50:25.103882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count duplicate image_id occurrences\nduplicate_count = df.duplicated(subset=[\"image_id\"]).sum()\n\nprint(f\"Number of duplicate image_id entries: {duplicate_count}\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T19:52:08.324772Z","iopub.execute_input":"2025-03-21T19:52:08.325142Z","iopub.status.idle":"2025-03-21T19:52:08.334604Z","shell.execute_reply.started":"2025-03-21T19:52:08.325113Z","shell.execute_reply":"2025-03-21T19:52:08.333577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Remove duplicate image_id entries, keeping only the first occurrence\ndf_unique = df.drop_duplicates(subset=[\"image_id\"], keep=\"first\")\n\n# Verify the shape after removing duplicates\nprint(f\"Shape after removing duplicates: {df_unique.shape}\")\n\n# Save the cleaned DataFrame to a new CSV file (optional)\ndf_unique.to_csv(\"cleaned_train.csv\", index=False)\ndf_unique.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T20:28:32.980801Z","iopub.execute_input":"2025-03-21T20:28:32.981157Z","iopub.status.idle":"2025-03-21T20:28:33.031162Z","shell.execute_reply.started":"2025-03-21T20:28:32.981132Z","shell.execute_reply":"2025-03-21T20:28:33.030084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_unique.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T20:31:50.471691Z","iopub.execute_input":"2025-03-21T20:31:50.472065Z","iopub.status.idle":"2025-03-21T20:31:50.47701Z","shell.execute_reply.started":"2025-03-21T20:31:50.472037Z","shell.execute_reply":"2025-03-21T20:31:50.475971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\nimport pandas as pd\n\n# Define disease classes\ndisease_classes = [\n    'No finding', 'Cardiomegaly', 'Aortic enlargement', 'Pleural thickening',\n    'ILD', 'Nodule/Mass', 'Pulmonary fibrosis', 'Lung Opacity', 'Atelectasis',\n    'Other lesion', 'Infiltration', 'Pleural effusion', 'Calcification',\n    'Consolidation', 'Pneumothorax'\n]\n\n# Define source and destination directories\nsource_dir = \"/kaggle/input/vinbigdata-original-image-dataset/vinbigdata/train\"  # Train images path\ndest_dir = \"sorted_images/\"  # Root folder to store categorized images\nos.makedirs(dest_dir, exist_ok=True)\n\n# Read CSV containing image IDs and labels\ndata = pd.read_csv(\"/kaggle/working/cleaned_train.csv\")  # Ensure CSV has 'image_id' and 'class_name' columns\n\n# Create subdirectories for each disease category\nfor disease in disease_classes:\n    os.makedirs(os.path.join(dest_dir, disease), exist_ok=True)\n\n# Move images to respective folders\nfor _, row in data.iterrows():\n    image_filename = row['image_id'] + \".jpg\"  # Assuming images have .jpg extension\n    image_path = os.path.join(source_dir, image_filename)\n    label = row['class_name']\n    \n    if label in disease_classes:\n        target_folder = os.path.join(dest_dir, label)\n        os.makedirs(target_folder, exist_ok=True)\n        if os.path.exists(image_path):\n            shutil.copy(image_path, os.path.join(target_folder, image_filename))\n        else:\n            print(f\"Warning: {image_path} not found.\")\n\nprint(\"Image sorting complete!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T20:12:47.097179Z","iopub.execute_input":"2025-03-21T20:12:47.097529Z","iopub.status.idle":"2025-03-21T20:19:13.82159Z","shell.execute_reply.started":"2025-03-21T20:12:47.097499Z","shell.execute_reply":"2025-03-21T20:19:13.820239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\nimport pandas as pd\n\n# Define disease classes\ndisease_classes = [\n    'No finding', 'Cardiomegaly', 'Aortic enlargement', 'Pleural thickening',\n    'ILD', 'Nodule/Mass', 'Pulmonary fibrosis', 'Lung Opacity', 'Atelectasis',\n    'Other lesion', 'Infiltration', 'Pleural effusion', 'Calcification',\n    'Consolidation', 'Pneumothorax'\n]\n\n# Define source and destination directories\nsource_dir = \"/kaggle/input/vinbigdata-original-image-dataset/vinbigdata/train\"  # Train images path\ndest_dir = \"/kaggle/working/sorted_images/\"  # Root folder to store categorized images\nos.makedirs(dest_dir, exist_ok=True)\n\n# Read CSV containing image IDs and labels\ndata = pd.read_csv(\"/kaggle/working/cleaned_train.csv\")  # Ensure CSV has 'image_id' and 'class_name' columns\n\n# Filter and segregate images\ndisease_counts = data['class_name'].value_counts()\nfor disease in disease_classes:\n    if disease in disease_counts and disease_counts[disease] >= 200:\n        target_folder = os.path.join(dest_dir, disease)\n        os.makedirs(target_folder, exist_ok=True)\n        \n        disease_subset = data[data['class_name'] == disease].head(200)\n        \n        for _, row in disease_subset.iterrows():\n            image_filename = row['image_id'] + \".jpg\"\n            image_path = os.path.join(source_dir, image_filename)\n            if os.path.exists(image_path):\n                shutil.copy(image_path, os.path.join(target_folder, image_filename))\n            else:\n                print(f\"Warning: {image_path} not found.\")\n\n# Count images in each folder\nimage_counts = {}\nfor disease in disease_classes:\n    folder_path = os.path.join(dest_dir, disease)\n    if os.path.exists(folder_path):\n        image_counts[disease] = len([f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))])\n    else:\n        image_counts[disease] = 0\n\n# Print image counts\nfor disease, count in image_counts.items():\n    print(f\"{disease}: {count} images\")\n\nprint(\"Image extraction and counting complete!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T20:40:50.512155Z","iopub.execute_input":"2025-03-21T20:40:50.512555Z","iopub.status.idle":"2025-03-21T20:41:31.848539Z","shell.execute_reply.started":"2025-03-21T20:40:50.512527Z","shell.execute_reply":"2025-03-21T20:41:31.846704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nsorted_images_dir = \"/kaggle/working/sorted_images\"\n\n# Get list of all folders and count images in each\nfolder_counts = {}\nfor folder in os.listdir(sorted_images_dir):\n    folder_path = os.path.join(sorted_images_dir, folder)\n    if os.path.isdir(folder_path):  # Ensure it's a directory\n        folder_counts[folder] = len([f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))])\n\n# Print the counts\nfor folder, count in folder_counts.items():\n    print(f\"{folder}: {count} images\")\n\n# Print total number of images\ntotal_images = sum(folder_counts.values())\nprint(f\"Total images in sorted_images: {total_images}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T20:43:08.423795Z","iopub.execute_input":"2025-03-21T20:43:08.424214Z","iopub.status.idle":"2025-03-21T20:43:08.445136Z","shell.execute_reply.started":"2025-03-21T20:43:08.424185Z","shell.execute_reply":"2025-03-21T20:43:08.444215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\nimport pandas as pd\n\n# Load the CSV file\ndf = pd.read_csv(\"/kaggle/working/cleaned_train.csv\")\n\n# Count occurrences of each unique disease\ndisease_counts = df[\"class_name\"].value_counts()\n\n# Filter diseases with at least 200 images\nselected_diseases = disease_counts[disease_counts >= 200].index.tolist()\n\n# Define paths\nimage_folder = \"/kaggle/input/vinbigdata-original-image-dataset/vinbigdata/train\"\ndestination_folder = \"/kaggle/working/filtered_images\"\n\n# Create main destination folder\nos.makedirs(destination_folder, exist_ok=True)\n\n# Process each selected disease\nfor disease in selected_diseases:\n    disease_folder = os.path.join(destination_folder, disease)\n    os.makedirs(disease_folder, exist_ok=True)\n\n    # Get image names for the disease\n    disease_images = df[df[\"class_name\"] == disease][\"path\"].tolist()\n\n    # Move images to their respective folders\n    for img in disease_images:\n        src_path = os.path.join(image_folder, img)\n        dest_path = os.path.join(disease_folder, os.path.basename(img))\n        \n        if os.path.exists(src_path):\n            shutil.copy(src_path, dest_path)  # Use shutil.move() if you want to move instead of copy\n\nprint(\"Segregation complete!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T20:38:27.277082Z","iopub.execute_input":"2025-03-21T20:38:27.277459Z","iopub.status.idle":"2025-03-21T20:38:27.32977Z","shell.execute_reply.started":"2025-03-21T20:38:27.277429Z","shell.execute_reply":"2025-03-21T20:38:27.32845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nfrom torch.utils.data import DataLoader\nfrom torchvision.datasets import ImageFolder\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\nimport torch_xla\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\n\n# Define device (Switch to TPU)\ndevice = xm.xla_device()\n\n# Hyperparameters\nNUM_CLASSES = 8\nBATCH_SIZE = 32\nEPOCHS = 10","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T20:57:01.437138Z","iopub.execute_input":"2025-03-21T20:57:01.437356Z","iopub.status.idle":"2025-03-21T20:57:40.464769Z","shell.execute_reply.started":"2025-03-21T20:57:01.437333Z","shell.execute_reply":"2025-03-21T20:57:40.463037Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define General Classifier\nclass GeneralClassifier(nn.Module):\n    def __init__(self, num_classes=NUM_CLASSES):\n        super(GeneralClassifier, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(1, 32, kernel_size=3, padding=1), nn.ReLU(),\n            nn.MaxPool2d(2, 2),\n            nn.Conv2d(32, 64, kernel_size=3, padding=1), nn.ReLU(),\n            nn.MaxPool2d(2, 2),\n            nn.Conv2d(64, 128, kernel_size=3, padding=1), nn.ReLU(),\n            nn.MaxPool2d(2, 2)\n        )\n        self.fc = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(128 * 28 * 28, 256), nn.ReLU(),\n            nn.Linear(256, num_classes)\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.fc(x)\n        return x\n\n# Load dataset\ntransform = transforms.Compose([\n    transforms.Grayscale(num_output_channels=1),\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.5], std=[0.5])\n])\n\nroot_path = \"/kaggle/working/sorted_images\"\n\ndataset = ImageFolder(root=r\"/kaggle/working/sorted_images\", transform=transform) \ndataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)\n\n# Model and optimizer\ngeneral_model = GeneralClassifier().to(device)\noptimizer_general = optim.Adam(general_model.parameters(), lr=0.001)\n\n# Training loop\nall_preds = []\nall_labels = []\nloss_fn = nn.CrossEntropyLoss()\n\nfor epoch in range(EPOCHS):\n    general_model.train()\n    epoch_loss = 0\n    correct = 0\n    total = 0\n\n    para_loader = pl.MpDeviceLoader(dataloader, device)  # TPU-compatible DataLoader\n    for images, labels in para_loader:\n        images, labels = images.to(device), labels.to(device)\n\n        optimizer_general.zero_grad()\n        outputs = general_model(images)\n\n        # Fix: Ensure labels are in correct range\n        if labels.max() >= NUM_CLASSES:\n            print(\"Error: Labels contain invalid values\")\n            break\n\n        loss = loss_fn(outputs, labels)\n        loss.backward()\n        xm.optimizer_step(optimizer_general)  # TPU-optimized step\n\n        epoch_loss += loss.item()\n        _, predicted = torch.max(outputs, 1)\n        correct += (predicted == labels).sum().item()\n        total += labels.size(0)\n\n        all_preds.extend(predicted.cpu().numpy())\n        all_labels.extend(labels.cpu().numpy())\n\n    xm.master_print(f\"Epoch {epoch+1}/{EPOCHS}, Loss: {epoch_loss/len(dataloader):.4f}, Accuracy: {correct/total:.4f}\")\n\n# Calculate precision, recall, and F1-score\naccuracy = accuracy_score(all_labels, all_preds)\nprecision = precision_score(all_labels, all_preds, average='macro')\nrecall = recall_score(all_labels, all_preds, average='macro')\nf1 = f1_score(all_labels, all_preds, average='macro')\n\nxm.master_print(f\"Final Accuracy: {accuracy:.4f}\")\nxm.master_print(f\"Final Precision: {precision:.4f}\")\nxm.master_print(f\"Final Recall: {recall:.4f}\")\nxm.master_print(f\"Final F1-score: {f1:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-21T21:08:23.188473Z","iopub.execute_input":"2025-03-21T21:08:23.188822Z","iopub.status.idle":"2025-03-21T21:08:23.259078Z","shell.execute_reply.started":"2025-03-21T21:08:23.188797Z","shell.execute_reply":"2025-03-21T21:08:23.257994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}