{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nimport scipy.ndimage\n\nprint(\"=========================================================\")\nprint(\"🚀 STARTING CLOUD TRAINING: FINAL ROBUST MODEL\")\nprint(\"=========================================================\")\n\n# ---------------------------------------------------------\n# 1. PATHS AND DATA LOADING\n# ---------------------------------------------------------\nCSV_PATH = '/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection/train.csv'\nIMAGE_DIR = '/kaggle/input/datasets/xhlulu/vinbigdata-chest-xray-resized-png-256x256/train/'\n\ndf_raw = pd.read_csv(CSV_PATH)\ndf_clean = df_raw[df_raw['class_id'].isin([8, 14])].copy()\ndf_clean['has_nodule'] = df_clean['class_id'].apply(lambda x: 1.0 if x == 8 else 0.0)\ndf_clean.fillna({'x_min': 0, 'y_min': 0, 'x_max': 0, 'y_max': 0}, inplace=True)\ndf_clean['image_path'] = df_clean['image_id'].apply(lambda x: os.path.join(IMAGE_DIR, f\"{x}.png\"))\n\nprint(f\"✅ Dataset loaded: {len(df_clean)} total images.\")\n\n# ---------------------------------------------------------\n# 2. FEATURE ENGINEERING & CLASS WEIGHTS\n# ---------------------------------------------------------\nIMG_SIZE = 256.0 \ndf_clean['x_center'] = ((df_clean['x_min'] + df_clean['x_max']) / 2.0) / IMG_SIZE\ndf_clean['y_center'] = ((df_clean['y_min'] + df_clean['y_max']) / 2.0) / IMG_SIZE\n\n# We use the entire dataset, no longer just a sample!\ndf_model = df_clean.copy() \n\nsani_count = len(df_model[df_model['has_nodule'] == 0.0])\nmalati_count = len(df_model[df_model['has_nodule'] == 1.0])\ntotal = sani_count + malati_count\nweight_0 = (1 / sani_count) * (total / 2.0)\nweight_1 = (1 / malati_count) * (total / 2.0)\ndf_model['sample_weight'] = df_model['has_nodule'].apply(lambda x: weight_1 if x == 1.0 else weight_0)\n\ndf_train, df_val = train_test_split(df_model, test_size=0.2, random_state=42, stratify=df_model['has_nodule'])\nprint(f\"✅ Split completed: {len(df_train)} Train | {len(df_val)} Validation.\")\n\n# ---------------------------------------------------------\n# 3. SOFT MASK CREATION (ANTI EDGE-EFFECT)\n# ---------------------------------------------------------\nmask_np = np.ones((224, 224, 3), dtype=np.float32)\nmask_np[:45, :45, :] = 0.0\nmask_np[:45, -45:, :] = 0.0\n\n# Heavy Gaussian Blur to avoid visual sharp edges (edge-effect)\nmask_np[:, :, 0] = scipy.ndimage.gaussian_filter(mask_np[:, :, 0], sigma=8)\nmask_np[:, :, 1] = scipy.ndimage.gaussian_filter(mask_np[:, :, 1], sigma=8)\nmask_np[:, :, 2] = scipy.ndimage.gaussian_filter(mask_np[:, :, 2], sigma=8)\n\ntf_corner_mask = tf.constant(mask_np)\n\n# ---------------------------------------------------------\n# 4. TF.DATA PIPELINE (DESTROYING ZOOM & LETTER BIASES)\n# ---------------------------------------------------------\ndef parse_function_train_mask(filename, label, x, y, weight):\n    image = tf.io.read_file(filename)\n    image = tf.image.decode_png(image, channels=3)\n    \n    # DESTROYING ZOOM BIAS: Random Cropping\n    image = tf.image.resize(image, [256, 256])\n    image = tf.image.random_crop(image, size=[224, 224, 3])\n    \n    # DESTROYING LETTER BIAS: Soft Masking\n    image = image * tf_corner_mask \n    \n    # DESTROYING LIGHTING BIAS: Color Augmentation\n    image = tf.image.random_brightness(image, max_delta=0.2)\n    image = tf.image.random_contrast(image, lower=0.8, upper=1.2)\n    \n    image = tf.keras.applications.resnet50.preprocess_input(image)\n    loc_target = tf.stack([label, x, y], axis=0)\n    return image, {'class_out': label, 'loc_out': loc_target}, {'class_out': weight, 'loc_out': 1.0}\n\ndef parse_function_val_mask(filename, label, x, y, weight):\n    image = tf.io.read_file(filename)\n    image = tf.image.decode_png(image, channels=3)\n    \n    # In validation, no random crops, we go straight to the center\n    image = tf.image.resize(image, [224, 224])\n    image = image * tf_corner_mask \n    \n    image = tf.keras.applications.resnet50.preprocess_input(image)\n    loc_target = tf.stack([label, x, y], axis=0)\n    return image, {'class_out': label, 'loc_out': loc_target}, {'class_out': weight, 'loc_out': 1.0}\n\ntrain_dataset = tf.data.Dataset.from_tensor_slices((\n    df_train['image_path'].values, df_train['has_nodule'].values,\n    df_train['x_center'].values, df_train['y_center'].values, df_train['sample_weight'].values\n)).map(parse_function_train_mask, num_parallel_calls=tf.data.AUTOTUNE).shuffle(1000).batch(32).prefetch(tf.data.AUTOTUNE)\n\nval_dataset = tf.data.Dataset.from_tensor_slices((\n    df_val['image_path'].values, df_val['has_nodule'].values,\n    df_val['x_center'].values, df_val['y_center'].values, df_val['sample_weight'].values\n)).map(parse_function_val_mask, num_parallel_calls=tf.data.AUTOTUNE).batch(32).prefetch(tf.data.AUTOTUNE)\n\n# ---------------------------------------------------------\n# 5. MODEL ARCHITECTURE AND COMPILATION\n# ---------------------------------------------------------\ndef masked_mse_loss(y_true, y_pred):\n    true_class = y_true[:, 0:1]\n    true_coords = y_true[:, 1:3]\n    squared_errors = tf.square(true_coords - y_pred)\n    mse_per_sample = tf.reduce_mean(squared_errors, axis=-1, keepdims=True)\n    return tf.reduce_mean(mse_per_sample * true_class)\n\nbase_model = tf.keras.applications.ResNet50(weights='imagenet', include_top=False, input_shape=(224, 224, 3))\nbase_model.trainable = False\n\nx = layers.GlobalAveragePooling2D()(base_model.output)\nx = layers.Dense(256, activation='relu')(x)\nx = layers.Dropout(0.5)(x)\n\nclass_output = layers.Dense(1, activation='sigmoid', name='class_out')(x)\nloc_output = layers.Dense(2, activation='linear', name='loc_out')(x)\n\nmodel_mask = models.Model(inputs=base_model.input, outputs={'class_out': class_output, 'loc_out': loc_output})\n\nmodel_mask.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),\n    loss={'class_out': 'binary_crossentropy', 'loc_out': masked_mse_loss},\n    loss_weights={'class_out': 1.0, 'loc_out': 1.0},\n    metrics={'class_out': ['accuracy']} \n)\n\n# ---------------------------------------------------------\n# 6. CALLBACKS AND TRAINING\n# ---------------------------------------------------------\ncallbacks = [\n    ModelCheckpoint(filepath='robust_nodule_model.keras', monitor='val_loss', save_best_only=True, verbose=1),\n    EarlyStopping(monitor='val_loss', patience=8, restore_best_weights=True, verbose=1),\n    ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=1e-6, verbose=1)\n]\n\nprint(\"Starting training on tens of thousands of radiographs...\")\n\n\n# CLOUD TRAINING EXECUTION BLOCK (Commented out for the master notebook)\ntry:\n    history = model_mask.fit(\n        train_dataset, \n        validation_data=val_dataset, \n        epochs=50, \n        callbacks=callbacks,\n        verbose=2 # Verbose 2 is cleaner for background Kaggle text logs\n    )\n    print(\"✅ Training completed successfully!\")\nexcept Exception as e:\n    import traceback\n    print(\"❌ ERROR CAUGHT DURING TRAINING:\")\n    traceback.print_exc()\n\nprint(\"=========================================================\")\nprint(\"OPERATION COMPLETED.\")\nprint(\"=========================================================\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}