{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594},{"sourceType":"datasetVersion","sourceId":1799839,"datasetId":1069682,"databundleVersionId":1837296},{"sourceType":"datasetVersion","sourceId":1800066,"datasetId":1069809,"databundleVersionId":1837523},{"sourceType":"datasetVersion","sourceId":1800067,"datasetId":1069810,"databundleVersionId":1837524},{"sourceType":"datasetVersion","sourceId":1800825,"datasetId":1070245,"databundleVersionId":1838284},{"sourceType":"datasetVersion","sourceId":1800777,"datasetId":1069787,"databundleVersionId":1838236},{"sourceType":"datasetVersion","sourceId":1800778,"datasetId":1070222,"databundleVersionId":1838237}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install numpy\n!pip install torch torchvision torchaudio\n!pip install ultralytics","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:27:42.217424Z","iopub.execute_input":"2024-12-06T17:27:42.218075Z","iopub.status.idle":"2024-12-06T17:28:08.732904Z","shell.execute_reply.started":"2024-12-06T17:27:42.218043Z","shell.execute_reply":"2024-12-06T17:28:08.731725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Carica il CSV\ntrain_df = pd.read_csv('/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv')\n\n# Analisi delle duplicazioni\nprint(\"\\nAnalisi delle annotazioni:\")\nprint(\"-\" * 50)\n\n# 1. Conta quante annotazioni ci sono per ogni combinazione immagine-classe\nduplicates = train_df.groupby(['image_id', 'class_name']).size().reset_index(name='count')\nduplicates_filtered = duplicates[duplicates['count'] > 1]\n\nprint(f\"Numero di combinazioni immagine-classe con annotazioni multiple: {len(duplicates_filtered)}\")\n\n# 2. Mostra alcuni esempi\nif len(duplicates_filtered) > 0:\n    print(\"\\nEsempi di annotazioni multiple:\")\n    print(duplicates_filtered.head())\n    \n    # 3. Mostra un esempio dettagliato di un'immagine con annotazioni multiple\n    example_image = duplicates_filtered.iloc[0]['image_id']\n    example_class = duplicates_filtered.iloc[0]['class_name']\n    print(f\"\\nDettaglio annotazioni per immagine {example_image}, classe {example_class}:\")\n    print(train_df[(train_df['image_id'] == example_image) & \n                  (train_df['class_name'] == example_class)])\n\n# 4. Statistiche generali\nprint(\"\\nStatistiche generali:\")\nprint(f\"Numero totale di immagini: {train_df['image_id'].nunique()}\")\nprint(f\"Numero totale di annotazioni: {len(train_df)}\")\nprint(\"\\nMedia annotazioni per immagine per classe:\")\nprint(duplicates['count'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:28:08.734754Z","iopub.execute_input":"2024-12-06T17:28:08.735081Z","iopub.status.idle":"2024-12-06T17:28:09.316982Z","shell.execute_reply.started":"2024-12-06T17:28:08.735051Z","shell.execute_reply":"2024-12-06T17:28:09.316012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n# Carica il CSV\ntrain_df = pd.read_csv('/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv')\n\n# Conta il numero di istanze (annotazioni) per ogni classe\ninstances_per_class = train_df['class_name'].value_counts()\n\n# Conta il numero di immagini uniche per ogni classe\nimages_per_class = train_df.groupby('class_name')['image_id'].nunique().sort_values(ascending=False)\n\nprint(\"Statistiche per classe:\")\nprint(\"-\" * 70)\nprint(f\"{'Classe':20} {'Immagini':8} {'Istanze':8} {'Media istanze/immagine':20}\")\nprint(\"-\" * 70)\n\nfor class_name in images_per_class.index:\n    images = images_per_class[class_name]\n    instances = instances_per_class[class_name]\n    avg_instances = instances / images\n    print(f\"{class_name:20} {images:8d} {instances:8d} {avg_instances:20.2f}\")\n\n# Calcola statistiche generali\ntotal_images = train_df['image_id'].nunique()\ntotal_instances = len(train_df)\n\nprint(\"\\nStatistiche generali:\")\nprint(\"-\" * 50)\nprint(f\"Totale immagini nel dataset: {total_images}\")\nprint(f\"Totale istanze/annotazioni: {total_instances}\")\nprint(f\"Media istanze per immagine: {total_instances/total_images:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:28:09.318074Z","iopub.execute_input":"2024-12-06T17:28:09.318361Z","iopub.status.idle":"2024-12-06T17:28:09.444336Z","shell.execute_reply.started":"2024-12-06T17:28:09.318336Z","shell.execute_reply":"2024-12-06T17:28:09.44344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Carica il CSV\ntrain_df = pd.read_csv('/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv')\n\n# Conta immagini con \"No finding\" vs immagini con patologie\nnormal_images = train_df[train_df['class_name'] == 'No finding']['image_id'].nunique()\nabnormal_images = train_df[train_df['class_name'] != 'No finding']['image_id'].nunique()\n\nprint(f\"Analisi delle Immagini:\")\nprint(f\"Totale immagini uniche: {train_df['image_id'].nunique()}\")\nprint(f\"Immagini con 'No finding': {normal_images}\")\nprint(f\"Immagini con patologie: {abnormal_images}\")\n\n# Analisi multi-label\nprint(\"\\nAnalisi Multi-label:\")\nimage_labels = train_df.groupby('image_id')['class_name'].nunique()\nprint(f\"Media etichette per immagine: {image_labels.mean():.2f}\")\nprint(f\"Max etichette per immagine: {image_labels.max()}\")\n\n# Distribuzione delle annotazioni per radiologo\nprint(\"\\nAnnotazioni per radiologo:\")\nprint(train_df['rad_id'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:28:09.446151Z","iopub.execute_input":"2024-12-06T17:28:09.446451Z","iopub.status.idle":"2024-12-06T17:28:09.597792Z","shell.execute_reply.started":"2024-12-06T17:28:09.446423Z","shell.execute_reply":"2024-12-06T17:28:09.596885Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Carica il CSV\ntrain_df = pd.read_csv('/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv')\n\n# Crea un DataFrame che mostra per ogni immagine quali classi sono state annotate da quali radiologi\ndef analyze_radiologist_agreement():\n    # Raggruppa per immagine e controlla se ci sono diagnosi diverse\n    disagreements = []\n    \n    for image_id in train_df['image_id'].unique():\n        image_annotations = train_df[train_df['image_id'] == image_id]\n        \n        # Ottieni le classi uniche per questa immagine\n        unique_classes = image_annotations['class_name'].unique()\n        \n        if len(unique_classes) > 1:  # Se c'è più di una classe\n            radiologists = {}\n            for _, row in image_annotations.iterrows():\n                rad_id = row['rad_id']\n                class_name = row['class_name']\n                if rad_id not in radiologists:\n                    radiologists[rad_id] = []\n                radiologists[rad_id].append(class_name)\n            \n            # Controlla se i radiologi sono in disaccordo\n            classes_per_rad = set(tuple(sorted(classes)) for classes in radiologists.values())\n            if len(classes_per_rad) > 1:\n                disagreements.append({\n                    'image_id': image_id,\n                    'diagnoses': radiologists\n                })\n    \n    return disagreements\n\ndisagreements = analyze_radiologist_agreement()\n\nprint(f\"Numero totale di immagini con disaccordo tra radiologi: {len(disagreements)}\")\nprint(\"\\nEsempi di disaccordi:\")\nprint(\"-\" * 70)\n\n# Mostra i primi 5 esempi di disaccordo\nfor i, case in enumerate(disagreements[:5]):\n    print(f\"\\nImmagine {i+1}: {case['image_id']}\")\n    for rad_id, diagnoses in case['diagnoses'].items():\n        print(f\"Radiologo {rad_id}: {', '.join(diagnoses)}\")\n\n# Calcola statistiche sul disaccordo\ntotal_images = len(train_df['image_id'].unique())\ndisagreement_rate = (len(disagreements) / total_images) * 100\n\nprint(f\"\\nPercentuale di immagini con disaccordo: {disagreement_rate:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:28:09.598976Z","iopub.execute_input":"2024-12-06T17:28:09.599327Z","iopub.status.idle":"2024-12-06T17:29:29.328683Z","shell.execute_reply.started":"2024-12-06T17:28:09.599288Z","shell.execute_reply":"2024-12-06T17:29:29.327681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport shutil\nimport yaml\nfrom sklearn.model_selection import train_test_split\nfrom ultralytics import YOLO\n\nfrom datetime import datetime\nimport time","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:29:29.329779Z","iopub.execute_input":"2024-12-06T17:29:29.330047Z","iopub.status.idle":"2024-12-06T17:29:33.753422Z","shell.execute_reply.started":"2024-12-06T17:29:29.330022Z","shell.execute_reply":"2024-12-06T17:29:33.752746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport shutil\nimport yaml\nimport time\nfrom datetime import datetime\nfrom sklearn.model_selection import train_test_split\n\nclass YOLODatasetPreparer:\n    def __init__(self, train_csv, image_dir, output_dir):\n        print(f\"\\n[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Inizializzazione dataset preparer...\")\n        \n        self.train_csv = train_csv\n        self.image_dir = image_dir\n        self.output_dir = output_dir\n        \n        # Crea directory necessarie\n        os.makedirs(os.path.join(output_dir, 'images', 'train'), exist_ok=True)\n        os.makedirs(os.path.join(output_dir, 'images', 'val'), exist_ok=True)\n        os.makedirs(os.path.join(output_dir, 'labels', 'train'), exist_ok=True)\n        os.makedirs(os.path.join(output_dir, 'labels', 'val'), exist_ok=True)\n        \n        # Carica e pre-processa le annotazioni\n        print(f\"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Caricamento e pre-processing dei dati...\")\n        self.train_annotations = pd.read_csv(train_csv)\n        \n        # Normalizza le coordinate usando width e height\n        self.train_annotations['x_min'] = self.train_annotations.apply(lambda row: row.x_min/row.width if pd.notna(row.x_min) else row.x_min, axis=1)\n        self.train_annotations['y_min'] = self.train_annotations.apply(lambda row: row.y_min/row.height if pd.notna(row.y_min) else row.y_min, axis=1)\n        self.train_annotations['x_max'] = self.train_annotations.apply(lambda row: row.x_max/row.width if pd.notna(row.x_max) else row.x_max, axis=1)\n        self.train_annotations['y_max'] = self.train_annotations.apply(lambda row: row.y_max/row.height if pd.notna(row.y_max) else row.y_max, axis=1)\n        \n        # Gestione \"No finding\"\n        no_finding_images = self.train_annotations[\n            self.train_annotations['class_name'] == 'No finding'\n        ]['image_id'].unique()\n        \n        print(f\"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Processamento immagini 'No finding'...\")\n        no_finding_annotations = []\n        for img_id in no_finding_images:\n            if len(self.train_annotations[\n                (self.train_annotations['image_id'] == img_id) & \n                (self.train_annotations['class_name'] != 'No finding')\n            ]) == 0:\n                no_finding_annotations.append({\n                    'image_id': img_id,\n                    'class_name': 'No finding',\n                    'x_min': 0,\n                    'y_min': 0,\n                    'x_max': 1,\n                    'y_max': 1,\n                    'width': 1024,\n                    'height': 1024\n                })\n        \n        if no_finding_annotations:\n            self.train_annotations = pd.concat([\n                self.train_annotations,\n                pd.DataFrame(no_finding_annotations)\n            ], ignore_index=True)\n        \n        # Crea mapping delle classi\n        self.class_map = self.create_class_mapping()\n        \n        # Crea split bilanciato per validation\n        self.create_validation_split()\n        \n        print(f\"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Inizializzazione completata\")\n\n    def create_class_mapping(self):\n        \"\"\"Crea un dizionario di mapping tra nomi delle classi e ID\"\"\"\n        unique_classes = sorted(self.train_annotations['class_name'].unique())\n        class_map = {cls: idx for idx, cls in enumerate(unique_classes)}\n        print(\"Class mapping:\", class_map)\n        return class_map\n    \n    def create_validation_split(self):\n        \"\"\"Crea uno split per validation mantenendo tutte le immagini nel training\"\"\"\n        self.train_annotations['split'] = 'train'  # Tutte le immagini per training\n        target_val_size = 4000\n        n_classes = len(self.class_map)\n        target_per_class = target_val_size // n_classes  # circa 267 immagini per classe\n        \n        val_images = set()  # Useremo un set per tenere traccia delle immagini di validation\n        \n        # Prima fase: selezione bilanciata per classe\n        print(\"\\nSelezione immagini per validation set:\")\n        for class_name in self.class_map.keys():\n            # Prendi gli image_id unici per questa classe\n            class_images = set(self.train_annotations[\n                self.train_annotations['class_name'] == class_name\n            ]['image_id'].unique())\n            \n            # Determina quante immagini prendere per questa classe\n            available = len(class_images)\n            available_new = len(class_images - val_images)  # Immagini non ancora selezionate\n            to_take = min(available_new, target_per_class)\n            \n            # Seleziona casualmente le immagini per questa classe\n            if to_take > 0:\n                # Se non abbiamo abbastanza immagini nuove, prendiamo anche da quelle già selezionate\n                if to_take > available_new:\n                    # Prima prendiamo tutte le nuove disponibili\n                    selected = class_images - val_images\n                    # Poi completiamo con immagini già selezionate se necessario\n                    additional_needed = to_take - len(selected)\n                    if additional_needed > 0:\n                        already_selected = class_images.intersection(val_images)\n                        if already_selected:\n                            additional = set(np.random.choice(list(already_selected), \n                                                           size=min(additional_needed, len(already_selected)), \n                                                           replace=False))\n                            selected.update(additional)\n                else:\n                    # Se abbiamo abbastanza immagini nuove, prendiamo solo da quelle\n                    selected = set(np.random.choice(list(class_images - val_images), \n                                                 size=to_take, \n                                                 replace=False))\n                \n                val_images.update(selected)\n                actual_selected = len(selected)\n            else:\n                actual_selected = 0\n            \n            print(f\"{class_name:20s}: {actual_selected:4d} images (available: {available}, new: {available_new})\")\n        \n        # Seconda fase: riempimento casuale se necessario\n        remaining_needed = target_val_size - len(val_images)\n        if remaining_needed > 0:\n            print(f\"\\nAggiunta di {remaining_needed} immagini random per raggiungere {target_val_size}\")\n            \n            # Prendi tutte le immagini disponibili che non sono ancora nel validation set\n            all_images = set(self.train_annotations['image_id'].unique())\n            available_images = list(all_images - val_images)\n            \n            if available_images:  # Verifica che ci siano immagini disponibili\n                # Seleziona casualmente le immagini rimanenti necessarie\n                additional_images = set(np.random.choice(available_images, \n                                                      size=min(remaining_needed, len(available_images)), \n                                                      replace=False))\n                val_images.update(additional_images)\n                print(f\"Aggiunte {len(additional_images)} immagini random\")\n            else:\n                print(\"Non ci sono più immagini disponibili per il riempimento random\")\n        \n        # Marca le immagini selezionate per validation\n        self.train_annotations.loc[\n            self.train_annotations['image_id'].isin(val_images),\n            'split'\n        ] = 'both'\n        \n        # Stampa statistiche finali\n        train_count = len(self.train_annotations['image_id'].unique())\n        val_count = len(val_images)\n        \n        print(\"\\nDistribuzione finale:\")\n        print(f\"Immagini totali: {train_count}\")\n        print(f\"Immagini nel training: {train_count} (100%)\")\n        print(f\"Immagini nel validation: {val_count}\")\n        \n        print(\"\\nDistribuzione per classe nel validation set:\")\n        for class_name in self.class_map.keys():\n            class_val = len(self.train_annotations[\n                (self.train_annotations['class_name'] == class_name) & \n                (self.train_annotations['split'] == 'both')\n            ]['image_id'].unique())\n            \n            print(f\"{class_name:20s}: {class_val:4d} images\")\n\n    def create_yolo_annotation(self, image_id):\n        \"\"\"Crea annotazioni nel formato YOLO\"\"\"\n        image_annotations = self.train_annotations[self.train_annotations['image_id'] == image_id]\n        \n        yolo_annotations = []\n        has_no_finding = False\n        \n        for _, row in image_annotations.iterrows():\n            try:\n                class_name = row['class_name']\n                \n                if class_name == 'No finding':\n                    if not has_no_finding:\n                        has_no_finding = True\n                        class_id = self.class_map[class_name]\n                        yolo_annotations.append(f\"{class_id} 0.5 0.5 0.9 0.9\")\n                    continue\n                \n                if pd.isna(row['x_min']) or pd.isna(row['y_min']) or \\\n                   pd.isna(row['x_max']) or pd.isna(row['y_max']):\n                    continue\n                \n                # Calcola centro e dimensioni\n                x_center = (row['x_min'] + row['x_max']) / 2\n                y_center = (row['y_min'] + row['y_max']) / 2\n                w = row['x_max'] - row['x_min']\n                h = row['y_max'] - row['y_min']\n                \n                if 0 <= x_center <= 1 and 0 <= y_center <= 1 and w > 0 and h > 0:\n                    class_id = self.class_map[class_name]\n                    annotation = f\"{class_id} {x_center:.6f} {y_center:.6f} {w:.6f} {h:.6f}\"\n                    yolo_annotations.append(annotation)\n            \n            except Exception as e:\n                print(f\"Error processing {class_name} in {image_id}: {str(e)}\")\n                continue\n        \n        return yolo_annotations\n\n    def _process_split(self, image_ids, split):\n        \"\"\"Processa immagini e label per train o validation\"\"\"\n        processed = 0\n        start_time = time.time()\n        \n        for image_id in image_ids:\n            # Percorsi dei file\n            png_path = os.path.join(self.image_dir, f\"{image_id}.png\")\n            label_path = os.path.join(\n                self.output_dir, 'labels', split, f\"{image_id}.txt\"\n            )\n            dest_image_path = os.path.join(\n                self.output_dir, 'images', split, f\"{image_id}.png\"\n            )\n            \n            try:\n                # Copia immagine\n                shutil.copy(png_path, dest_image_path)\n                \n                # Crea e salva annotazioni YOLO\n                yolo_annotations = self.create_yolo_annotation(image_id)\n                if yolo_annotations:\n                    with open(label_path, 'w') as f:\n                        f.write('\\n'.join(yolo_annotations))\n                \n                processed += 1\n                if processed % 1000 == 0:\n                    elapsed_time = time.time() - start_time\n                    speed = processed / elapsed_time\n                    print(f\"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] \"\n                          f\"Processed {processed}/{len(image_ids)} images for {split} \"\n                          f\"({speed:.1f} img/s)\")\n                \n            except Exception as e:\n                print(f\"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] \"\n                      f\"Error processing {image_id}: {str(e)}\")\n                continue\n\n    def prepare_yolo_dataset(self):\n        \"\"\"Prepara il dataset YOLO usando lo split bilanciato\"\"\"\n        # Pulisci le directory di output\n        train_dir = os.path.join(self.output_dir, 'images', 'train')\n        val_dir = os.path.join(self.output_dir, 'images', 'val')\n        train_labels = os.path.join(self.output_dir, 'labels', 'train')\n        val_labels = os.path.join(self.output_dir, 'labels', 'val')\n        \n        for dir_path in [train_dir, val_dir, train_labels, val_labels]:\n            if os.path.exists(dir_path):\n                shutil.rmtree(dir_path)\n            os.makedirs(dir_path)\n        \n        # Get image IDs for each split\n        train_ids = self.train_annotations['image_id'].unique()  # Tutte le immagini per training\n        val_ids = self.train_annotations[\n            self.train_annotations['split'] == 'both'\n        ]['image_id'].unique()\n        \n        print(f\"\\n[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Processing splits\")\n        print(f\"Training images: {len(train_ids)}\")\n        print(f\"Validation images: {len(val_ids)}\")\n        \n        # Processa gli split\n        self._process_split(train_ids, 'train')\n        self._process_split(val_ids, 'val')\n        \n        # Crea file YAML\n        yaml_config = {\n            'path': self.output_dir,\n            'train': 'images/train',\n            'val': 'images/val',\n            'nc': len(self.class_map),\n            'names': list(self.class_map.keys())\n        }\n        \n        yaml_path = os.path.join(self.output_dir, 'dataset.yaml')\n        with open(yaml_path, 'w') as f:\n            yaml.dump(yaml_config, f, default_flow_style=False)\n        \n        print(f\"\\nConfig saved to {yaml_path}\")\n        return yaml_path","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:29:33.754571Z","iopub.execute_input":"2024-12-06T17:29:33.754983Z","iopub.status.idle":"2024-12-06T17:29:33.785019Z","shell.execute_reply.started":"2024-12-06T17:29:33.754956Z","shell.execute_reply":"2024-12-06T17:29:33.784062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preparer = YOLODatasetPreparer(\n    train_csv='/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train.csv',\n    image_dir='/kaggle/input/vinbigdata-1024-image-dataset/vinbigdata/train',\n    output_dir='/kaggle/working/yolo_dataset'\n)\n\nyaml_path = preparer.prepare_yolo_dataset()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:29:33.786129Z","iopub.execute_input":"2024-12-06T17:29:33.786393Z","iopub.status.idle":"2024-12-06T17:36:49.696052Z","shell.execute_reply.started":"2024-12-06T17:29:33.786363Z","shell.execute_reply":"2024-12-06T17:36:49.695132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import yaml\n\nyaml_path = '/kaggle/working/yolo_dataset/dataset.yaml'\n# Verifica il contenuto del file YAML\nwith open(yaml_path, 'r') as f:\n    config = yaml.safe_load(f)\n    \n# Verifica le immagini\nval_path = os.path.join(config['path'], config['val'])\ntrain_path = os.path.join(config['path'], config['train'])\n\nprint(f\"Immagini nel training set: {len(os.listdir(train_path))}\")\nprint(f\"Immagini nel validation set: {len(os.listdir(val_path))}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:36:49.697131Z","iopub.execute_input":"2024-12-06T17:36:49.697404Z","iopub.status.idle":"2024-12-06T17:36:49.716902Z","shell.execute_reply.started":"2024-12-06T17:36:49.697378Z","shell.execute_reply":"2024-12-06T17:36:49.716124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Stampe di alcune immagini con i relativi bounding box per verificare la corretta preparazione del dataset\n\nimport os\nimport cv2\nimport numpy as np\nfrom pathlib import Path\nimport random\nimport matplotlib.pyplot as plt\nfrom collections import defaultdict\n\ndef validate_yolo_dataset(dataset_path, samples_per_class=1):\n    \"\"\"\n    Valida il dataset YOLO visualizzando almeno un'immagine per ogni classe.\n    \n    Args:\n        dataset_path: percorso alla cartella del dataset\n        samples_per_class: numero minimo di immagini da visualizzare per classe\n    \"\"\"\n    # Classi nel corretto ordine\n    classes = {\n        0: 'Aortic enlargement',\n        1: 'Atelectasis',\n        2: 'Calcification',\n        3: 'Cardiomegaly',\n        4: 'Consolidation',\n        5: 'ILD',\n        6: 'Infiltration',\n        7: 'Lung Opacity',\n        8: 'No finding',\n        9: 'Nodule/Mass',\n        10: 'Other lesion',\n        11: 'Pleural effusion',\n        12: 'Pleural thickening',\n        13: 'Pneumothorax',\n        14: 'Pulmonary fibrosis'\n    }\n    \n    # Percorsi alle immagini e label\n    images_path = Path(dataset_path) / 'images' / 'train'\n    labels_path = Path(dataset_path) / 'labels' / 'train'\n    \n    # Verifica esistenza directory\n    if not images_path.exists() or not labels_path.exists():\n        raise ValueError(f\"Directory non trovate in {dataset_path}\")\n    \n    # Lista tutte le immagini e relative label\n    image_files = list(images_path.glob('*.jpg')) + list(images_path.glob('*.png'))\n    if not image_files:\n        raise ValueError(f\"Nessuna immagine trovata in {images_path}\")\n    \n    # Dizionario per tenere traccia delle immagini per classe\n    images_by_class = defaultdict(list)\n    \n    # Statistiche\n    stats = {\n        'total_images': len(image_files),\n        'missing_labels': 0,\n        'invalid_boxes': 0,\n        'boxes_per_class': {i: 0 for i in range(len(classes))}\n    }\n    \n    # Prima passata: categorizza le immagini per classe\n    for img_path in image_files:\n        label_path = labels_path / f\"{img_path.stem}.txt\"\n        if not label_path.exists():\n            stats['missing_labels'] += 1\n            continue\n            \n        with open(label_path) as f:\n            for line in f:\n                try:\n                    class_id = int(float(line.strip().split()[0]))\n                    images_by_class[class_id].append(img_path)\n                    stats['boxes_per_class'][class_id] += 1\n                except:\n                    continue\n    \n    # Seleziona immagini da visualizzare\n    images_to_show = []\n    for class_id in classes.keys():\n        if class_id in images_by_class:\n            # Prendi samples_per_class immagini per questa classe\n            selected = random.sample(\n                images_by_class[class_id],\n                min(samples_per_class, len(images_by_class[class_id]))\n            )\n            images_to_show.extend(selected)\n    \n    # Rimuovi duplicati mantenendo l'ordine\n    images_to_show = list(dict.fromkeys(images_to_show))\n    \n    print(f\"\\nMostrando {len(images_to_show)} immagini...\\n\")\n    \n    for img_path in images_to_show:\n        # Carica immagine\n        img = cv2.imread(str(img_path))\n        if img is None:\n            print(f\"Errore nel caricamento dell'immagine: {img_path}\")\n            continue\n            \n        height, width = img.shape[:2]\n        \n        # Trova il corrispondente file label\n        label_path = labels_path / f\"{img_path.stem}.txt\"\n        if not label_path.exists():\n            continue\n            \n        # Leggi le label\n        boxes = []\n        class_ids_in_image = set()\n        with open(label_path) as f:\n            for line in f:\n                try:\n                    class_id, x, y, w, h = map(float, line.strip().split())\n                    class_id = int(class_id)\n                    class_ids_in_image.add(class_id)\n                    \n                    if not (0 <= x <= 1 and 0 <= y <= 1 and 0 <= w <= 1 and 0 <= h <= 1):\n                        stats['invalid_boxes'] += 1\n                        continue\n                        \n                    # Converti da formato YOLO a pixel\n                    x1 = int((x - w/2) * width)\n                    y1 = int((y - h/2) * height)\n                    x2 = int((x + w/2) * width)\n                    y2 = int((y + h/2) * height)\n                    \n                    boxes.append((class_id, (x1, y1, x2, y2)))\n                except:\n                    print(f\"Errore nel parsing della label: {line.strip()} in {label_path}\")\n                    continue\n        \n        # Disegna i box\n        for class_id, (x1, y1, x2, y2) in boxes:\n            color = plt.cm.rainbow(class_id / len(classes))\n            color = tuple(int(255 * c) for c in color[:3])  # Converti in BGR\n            cv2.rectangle(img, (x1, y1), (x2, y2), color, 2)\n            cv2.putText(img, classes[class_id], (x1, y1-10), \n                       cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)\n        \n        # Mostra l'immagine\n        plt.figure(figsize=(15, 10))\n        plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n        plt.title(f\"Image: {img_path.name}\\nClassi presenti: {', '.join(classes[cid] for cid in class_ids_in_image)}\")\n        plt.axis('off')\n        plt.show()\n    \n    # Stampa statistiche\n    print(\"\\nStatistiche Dataset:\")\n    print(f\"Totale immagini: {stats['total_images']}\")\n    print(f\"Label mancanti: {stats['missing_labels']}\")\n    print(f\"Box invalidi: {stats['invalid_boxes']}\")\n    print(\"\\nDistribuzione box per classe:\")\n    for class_id, count in stats['boxes_per_class'].items():\n        print(f\"{classes[class_id]}: {count}\")\n    \n    return stats\n\n# Uso:\ndataset_path = '/kaggle/working/yolo_dataset'\nstats = validate_yolo_dataset(dataset_path, samples_per_class=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:36:49.720028Z","iopub.execute_input":"2024-12-06T17:36:49.720275Z","iopub.status.idle":"2024-12-06T17:36:58.830567Z","shell.execute_reply.started":"2024-12-06T17:36:49.720251Z","shell.execute_reply":"2024-12-06T17:36:58.829771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Addestramento\nimport wandb\n#wandb.login(key='d0d96c9a2f07c81db580937bffde63589ab42fba')\n#wandb.init(mode=\"offline\")\nwandb.init(mode=\"disabled\")\n\nmodel = YOLO('/kaggle/working/runs/train/yolov11n_improved/weights/best.pt')\n# model = YOLO('yolo11n.pt')\nresults = model.train(\n    data=yaml_path,\n    epochs=1,                    \n    imgsz=1024,                  \n    batch=16,                   \n    device=0,\n    cache=False,                  \n    patience=20,               \n    plots=True,\n    save_period=5,              \n    workers=6,                  \n    seed=42,\n    resume=False,\n    project='/kaggle/working/runs/train',\n    name='yolov11n_improved',    \n    exist_ok=True,            \n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T17:36:58.831962Z","iopub.execute_input":"2024-12-06T17:36:58.832538Z","iopub.status.idle":"2024-12-06T17:46:47.536342Z","shell.execute_reply.started":"2024-12-06T17:36:58.832498Z","shell.execute_reply":"2024-12-06T17:46:47.535227Z"}},"outputs":[],"execution_count":null}]}