{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":51753,"databundleVersionId":5692552,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import necessary libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tqdm import tqdm\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"TensorFlow version:\", tf.__version__)\n\ndef create_optimized_optimizer(learning_rate=3e-4):\n    \"\"\"\n    Create AdamW optimizer with better defaults inspired by Ranger\n    AdamW combines Adam with decoupled weight decay (similar to RAdam benefits)\n    \"\"\"\n    return keras.optimizers.AdamW(\n        learning_rate=learning_rate,\n        beta_1=0.95,  # Higher momentum (Ranger recommendation)\n        beta_2=0.999,\n        weight_decay=0.01,  # Decoupled weight decay\n        clipnorm=1.0  # Gradient clipping for stability\n    )\n\n# Memory-efficient configuration\nCONFIG = {\n    'img_height': 256,\n    'img_width': 256,\n    'batch_size': 4,\n    'epochs': 50,  # More epochs for better convergence\n    'learning_rate': 5e-4,  # Slightly higher\n    'random_seed': 42,\n    'max_samples': 800,  # More training data\n    'min_contrail_pixels': 80,  # Slightly lower threshold for more variety\n}\n\ntf.random.set_seed(CONFIG['random_seed'])\nnp.random.seed(CONFIG['random_seed'])\n\n# Enable memory growth\ngpus = tf.config.list_physical_devices('GPU')\nif gpus:\n    for gpu in gpus:\n        tf.config.experimental.set_memory_growth(gpu, True)\n\nprint(\"Loading metadata...\")\nwith open(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train_metadata.json\") as f:\n    train_meta = json.load(f)\n\nBASE_PATH = \"/kaggle/input/google-research-identify-contrails-reduce-global-warming\"\nTRAIN_PATH = os.path.join(BASE_PATH, \"train\")\n\ndef normalize_range(data):\n    \"\"\"Better normalization for satellite data\"\"\"\n    data = data.astype(np.float32)\n    # Use mean/std instead of percentile for better stability\n    mean = np.mean(data)\n    std = np.std(data)\n    if std > 0:\n        return (data - mean) / (std + 1e-8)\n    return data - mean\n\ndef load_sample_memory_efficient(record_id):\n    \"\"\"Load with minimal memory footprint\"\"\"\n    record_path = os.path.join(TRAIN_PATH, record_id)\n    \n    try:\n        # Load only essential bands\n        band15 = np.load(os.path.join(record_path, \"band_15.npy\"))\n        band14 = np.load(os.path.join(record_path, \"band_14.npy\"))\n        band11 = np.load(os.path.join(record_path, \"band_11.npy\"))\n        mask = np.load(os.path.join(record_path, \"human_pixel_masks.npy\"))\n        \n        # Use middle timestep and neighbors\n        n_times = band15.shape[2]\n        mid = n_times // 2\n        \n        # Get 3 timesteps around middle\n        if n_times >= 3:\n            times = [max(0, mid-1), mid, min(n_times-1, mid+1)]\n        else:\n            times = [mid] * 3\n        \n        channels = []\n        for t in times:\n            b15 = normalize_range(band15[:, :, t])\n            b14 = normalize_range(band14[:, :, t])\n            b11 = normalize_range(band11[:, :, t])\n            btd = b15 - b14  # Most important feature!\n            \n            channels.extend([btd, b15, b11])  # BTD first!\n        \n        image = np.stack(channels[:9], axis=-1).astype(np.float32)\n        \n        # Process mask - take maximum across all annotations\n        if len(mask.shape) == 3:\n            mask = np.max(mask, axis=-1)\n        mask = mask.astype(np.float32)\n        \n        # Resize with better interpolation\n        image = tf.image.resize(image, [CONFIG['img_height'], CONFIG['img_width']], \n                               method='bilinear').numpy()\n        mask = tf.image.resize(np.expand_dims(mask, -1), \n                              [CONFIG['img_height'], CONFIG['img_width']],\n                              method='nearest').numpy()  # Use nearest for masks!\n        mask = (mask > 0.5).astype(np.float32)\n        \n        # Clean up\n        del band15, band14, band11\n        \n        return image, mask\n        \n    except Exception as e:\n        return None, None\n\n# Generator for memory efficiency\ndef data_generator(metadata, batch_size, max_samples, augment=False):\n    \"\"\"Memory-efficient data generator\"\"\"\n    \n    while True:\n        batch_images = []\n        batch_masks = []\n        samples_in_epoch = 0\n        \n        # Shuffle metadata each epoch\n        indices = list(range(len(metadata)))\n        np.random.shuffle(indices)\n        \n        for idx in indices:\n            if samples_in_epoch >= max_samples:\n                break\n                \n            meta = metadata[idx]\n            image, mask = load_sample_memory_efficient(meta['record_id'])\n            \n            # Only use samples with significant contrails\n            if image is not None and np.sum(mask) > CONFIG['min_contrail_pixels']:\n                if augment and np.random.rand() > 0.5:\n                    # Random rotation\n                    k = np.random.randint(0, 4)\n                    image = np.rot90(image, k)\n                    mask = np.rot90(mask, k)\n                    \n                    # Random flip\n                    if np.random.rand() > 0.5:\n                        image = np.fliplr(image)\n                        mask = np.fliplr(mask)\n                \n                batch_images.append(image)\n                batch_masks.append(mask)\n                samples_in_epoch += 1\n                \n                if len(batch_images) == batch_size:\n                    yield (np.array(batch_images, dtype=np.float32), \n                           np.array(batch_masks, dtype=np.float32))\n                    batch_images = []\n                    batch_masks = []\n                    gc.collect()\n        \n        # Yield remaining samples if any\n        if len(batch_images) > 0:\n            yield (np.array(batch_images, dtype=np.float32), \n                   np.array(batch_masks, dtype=np.float32))\n            gc.collect()\n\n# Loss functions\ndef dice_coef(y_true, y_pred, smooth=1.0):\n    y_true = tf.cast(tf.squeeze(y_true), tf.float32)\n    y_pred = tf.cast(tf.squeeze(y_pred), tf.float32)\n    y_true_f = tf.reshape(y_true, [-1])\n    y_pred_f = tf.reshape(y_pred, [-1])\n    intersection = tf.reduce_sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (tf.reduce_sum(y_true_f) + tf.reduce_sum(y_pred_f) + smooth)\n\ndef dice_loss(y_true, y_pred):\n    return 1.0 - dice_coef(y_true, y_pred)\n\ndef combined_loss(y_true, y_pred):\n    \"\"\"Loss optimized for highly imbalanced data\"\"\"\n    y_true = tf.squeeze(y_true)\n    y_pred = tf.squeeze(y_pred)\n    \n    # Dice loss (main component)\n    dice = dice_loss(y_true, y_pred)\n    \n    # Focal loss (better than BCE for imbalance)\n    epsilon = 1e-7\n    y_pred_clipped = tf.clip_by_value(y_pred, epsilon, 1 - epsilon)\n    \n    # Focal loss with gamma=2\n    alpha = 0.75  # Weight for positive class\n    gamma = 2.0\n    \n    p_t = tf.where(y_true > 0.5, y_pred_clipped, 1 - y_pred_clipped)\n    alpha_t = tf.where(y_true > 0.5, alpha, 1 - alpha)\n    focal = -alpha_t * tf.pow(1 - p_t, gamma) * tf.math.log(p_t)\n    focal = tf.reduce_mean(focal)\n    \n    # Weighted combination\n    return 0.7 * dice + 0.3 * focal\n\n# Compact U-Net\ndef create_compact_unet(input_shape=(256, 256, 9)):\n    \"\"\"Memory-efficient U-Net with dropout\"\"\"\n    \n    def conv_block(x, filters, dropout_rate=0.2):\n        x = layers.Conv2D(filters, 3, padding='same')(x)\n        x = layers.BatchNormalization()(x)\n        x = layers.Activation('relu')(x)\n        x = layers.Dropout(dropout_rate)(x)\n        x = layers.Conv2D(filters, 3, padding='same')(x)\n        x = layers.BatchNormalization()(x)\n        x = layers.Activation('relu')(x)\n        return x\n    \n    inputs = layers.Input(shape=input_shape)\n    \n    # Encoder\n    c1 = conv_block(inputs, 32, 0.1)\n    p1 = layers.MaxPooling2D(2)(c1)\n    \n    c2 = conv_block(p1, 64, 0.1)\n    p2 = layers.MaxPooling2D(2)(c2)\n    \n    c3 = conv_block(p2, 128, 0.2)\n    p3 = layers.MaxPooling2D(2)(c3)\n    \n    # Bottleneck\n    c4 = conv_block(p3, 256, 0.3)\n    \n    # Decoder\n    u5 = layers.UpSampling2D(2)(c4)\n    u5 = layers.concatenate([u5, c3])\n    c5 = conv_block(u5, 128, 0.2)\n    \n    u6 = layers.UpSampling2D(2)(c5)\n    u6 = layers.concatenate([u6, c2])\n    c6 = conv_block(u6, 64, 0.1)\n    \n    u7 = layers.UpSampling2D(2)(c6)\n    u7 = layers.concatenate([u7, c1])\n    c7 = conv_block(u7, 32, 0.1)\n    \n    outputs = layers.Conv2D(1, 1, activation='sigmoid')(c7)\n    \n    return keras.Model(inputs, outputs, name='Compact_UNet')\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"MEMORY-EFFICIENT CONTRAIL DETECTION\")\nprint(\"=\"*60)\n\n# Create model\nprint(\"\\nCreating model...\")\nmodel = create_compact_unet(input_shape=(CONFIG['img_height'], CONFIG['img_width'], 9))\n\n# Use optimized AdamW instead of regular Adam\noptimizer = create_optimized_optimizer(learning_rate=CONFIG['learning_rate'])\n\nmodel.compile(\n    optimizer=optimizer,\n    loss=combined_loss,\n    metrics=[dice_coef, 'accuracy']\n)\n\nprint(model.summary())\n\n# Calculate steps\nsteps_per_epoch = CONFIG['max_samples'] // CONFIG['batch_size']\nvalidation_steps = 50 // CONFIG['batch_size']\n\nprint(f\"\\nSteps per epoch: {steps_per_epoch}\")\nprint(f\"Validation steps: {validation_steps}\")\n\n# Split metadata\nsplit_idx = int(len(train_meta) * 0.85)\ntrain_metadata = train_meta[:split_idx]\nval_metadata = train_meta[split_idx:]\n\n# Create generators\ntrain_gen = data_generator(train_metadata, CONFIG['batch_size'], CONFIG['max_samples'], augment=True)\nval_gen = data_generator(val_metadata, CONFIG['batch_size'], 50, augment=False)\n\n# Callbacks\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(\n        'best_model.h5',\n        monitor='val_dice_coef',\n        mode='max',\n        save_best_only=True,\n        verbose=1\n    ),\n    keras.callbacks.EarlyStopping(\n        monitor='val_dice_coef',\n        patience=10,\n        mode='max',\n        restore_best_weights=True,\n        verbose=1\n    ),\n    keras.callbacks.ReduceLROnPlateau(\n        monitor='val_loss',\n        factor=0.5,\n        patience=5,\n        min_lr=1e-7,\n        verbose=1\n    )\n]\n\n# Train\nprint(\"\\nStarting training...\")\nhistory = model.fit(\n    train_gen,\n    steps_per_epoch=steps_per_epoch,\n    validation_data=val_gen,\n    validation_steps=validation_steps,\n    epochs=CONFIG['epochs'],\n    callbacks=callbacks,\n    verbose=1\n)\n\n# Load best model\nmodel.load_weights('best_model.h5')\n\nprint(\"\\n=== TRAINING COMPLETED ===\")\nprint(f\"Best validation Dice: {max(history.history['val_dice_coef']):.4f}\")\n\n# Visualize predictions\nprint(\"\\nGenerating predictions...\")\nval_samples = []\nval_masks_gt = []\n\nfor meta in val_metadata[:6]:\n    img, msk = load_sample_memory_efficient(meta['record_id'])\n    if img is not None and np.sum(msk) > 50:\n        val_samples.append(img)\n        val_masks_gt.append(msk)\n    if len(val_samples) >= 6:\n        break\n\nif len(val_samples) > 0:\n    val_samples = np.array(val_samples)\n    val_masks_gt = np.array(val_masks_gt)\n    predictions = model.predict(val_samples, verbose=0)\n    \n    fig, axes = plt.subplots(len(val_samples), 4, figsize=(16, 4*len(val_samples)))\n    if len(val_samples) == 1:\n        axes = axes.reshape(1, -1)\n    \n    for i in range(len(val_samples)):\n        # BTD channel\n        axes[i, 0].imshow(val_samples[i, :, :, 2], cmap='RdBu_r')\n        axes[i, 0].set_title('BTD Input')\n        axes[i, 0].axis('off')\n        \n        # Ground truth\n        gt = val_masks_gt[i].squeeze()\n        axes[i, 1].imshow(gt, cmap='gray')\n        axes[i, 1].set_title(f'GT ({np.sum(gt):.0f} px)')\n        axes[i, 1].axis('off')\n        \n        # Prediction\n        pred = predictions[i].squeeze()\n        axes[i, 2].imshow(pred, cmap='hot', vmin=0, vmax=1)\n        axes[i, 2].set_title(f'Pred (max: {pred.max():.2f})')\n        axes[i, 2].axis('off')\n        \n        # Overlay\n        pred_bin = pred > 0.5\n        dice = 2 * np.sum(pred_bin * gt) / (np.sum(pred_bin) + np.sum(gt) + 1e-8)\n        \n        overlay = np.stack([val_samples[i, :, :, 2]]*3, axis=-1)\n        overlay = (overlay - overlay.min()) / (overlay.max() - overlay.min() + 1e-8)\n        \n        tp = np.logical_and(pred_bin, gt > 0)\n        fp = np.logical_and(pred_bin, gt == 0)\n        fn = np.logical_and(~pred_bin, gt > 0)\n        \n        overlay[tp] = [0, 1, 0]\n        overlay[fp] = [1, 0, 0]\n        overlay[fn] = [0, 0, 1]\n        \n        axes[i, 3].imshow(overlay)\n        axes[i, 3].set_title(f'Dice: {dice:.3f}')\n        axes[i, 3].axis('off')\n    \n    plt.tight_layout()\n    plt.savefig('predictions.png', dpi=100, bbox_inches='tight')\n    plt.show()\n\n# Plot history\nfig, axes = plt.subplots(1, 2, figsize=(12, 4))\n\naxes[0].plot(history.history['loss'], label='Train')\naxes[0].plot(history.history['val_loss'], label='Val')\naxes[0].set_title('Loss')\naxes[0].legend()\naxes[0].grid(True)\n\naxes[1].plot(history.history['dice_coef'], label='Train')\naxes[1].plot(history.history['val_dice_coef'], label='Val')\naxes[1].set_title('Dice Coefficient')\naxes[1].legend()\naxes[1].grid(True)\n\nplt.tight_layout()\nplt.savefig('training_history.png', dpi=100)\nplt.show()\n\nmodel.save('final_model.keras')\nprint(\"\\n✅ Model saved as 'final_model.keras'\")\n\n# Cleanup\ngc.collect()\nkeras.backend.clear_session()\n\nprint(\"\\n🎉 Training completed!\")\nprint(f\"Memory-efficient settings:\")\nprint(f\"  - Image size: {CONFIG['img_height']}x{CONFIG['img_width']}\")\nprint(f\"  - Batch size: {CONFIG['batch_size']}\")\nprint(f\"  - Max samples: {CONFIG['max_samples']}\")\nprint(f\"  - Input channels: 9 (3 timesteps × 3 features)\")\nprint(f\"  - Using data generator (no full dataset in memory)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T09:50:10.752953Z","iopub.execute_input":"2025-10-19T09:50:10.753937Z","iopub.status.idle":"2025-10-19T13:24:58.295873Z","shell.execute_reply.started":"2025-10-19T09:50:10.75391Z","shell.execute_reply":"2025-10-19T13:24:58.28702Z"}},"outputs":[],"execution_count":null}]}