{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":52950,"databundleVersionId":5973250,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":798592,"databundleVersionId":16228620,"modelInstanceId":608217,"modelId":620117,"isSourceIdPinned":false},{"sourceType":"kernelVersion","sourceId":301228313,"isSourceIdPinned":false}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"cell-md-0","cell_type":"markdown","source":"# ASL Fingerspelling v6 — Fine-tune v5 + Supplemental Data\n## Group 12 · MIE1517 · Kaggle T4\n\nLoads the **v5 Conformer-Transformer checkpoint** and fine-tunes it on:\n- **~67 K** NPY sequences from the main competition split (same 70/15/15 as v5)\n- **~50 K** supplemental fingerspelling sequences loaded from raw parquet (RAM-cached)\n\nKey changes over v5:\n| Feature | v5 | v6 (this notebook) |\n|---|---|---|\n| Training data | 48 K NPY only | 48 K NPY + 50 K supplemental |\n| Scheduled sampling start | epoch 8 | epoch 8 |\n| Checkpoint | `asl_transformer_v5_best.pth` | `asl_transformer_v6_best.pth` |\n| Resume token | `RESUME_FROM = CKPT_LAST` | same |\n","metadata":{}},{"id":"9de690ec-2291-4e10-896d-679e9436e890","cell_type":"markdown","source":"## 1. Setup & Configuration","metadata":{}},{"id":"cell-imports","cell_type":"code","source":"import os, math, json, time, random, warnings, subprocess, sys\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, ConcatDataset\nfrom torch.amp import GradScaler, autocast\n\ntry:\n    import editdistance\nexcept ImportError:\n    subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', 'editdistance'])\n    import editdistance\n\nSEED = 42\nrandom.seed(SEED); np.random.seed(SEED)\ntorch.manual_seed(SEED); torch.cuda.manual_seed_all(SEED)\ntorch.backends.cudnn.deterministic = True\n\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'GPU  {torch.cuda.get_device_name(0)}')\nprint(f'VRAM {torch.cuda.get_device_properties(0).total_memory/1e9:.1f} GB')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T02:48:12.96964Z","iopub.execute_input":"2026-03-23T02:48:12.970402Z","iopub.status.idle":"2026-03-23T02:48:17.789215Z","shell.execute_reply.started":"2026-03-23T02:48:12.970357Z","shell.execute_reply":"2026-03-23T02:48:17.788423Z"}},"outputs":[],"execution_count":null},{"id":"68cfd10f-dccb-449b-9dbd-0b9eb2ad5242","cell_type":"markdown","source":"## 2. Paths & Hyperparameters","metadata":{}},{"id":"cell-config","cell_type":"code","source":"class Config:\n    # ── Competition Data (Attach standard ASL fingerspelling dataset) ────────\n    COMP_DIR    = Path('/kaggle/input/competitions/asl-fingerspelling')\n    TRAIN_META  = COMP_DIR / 'train.csv'\n    SUPP_META   = COMP_DIR / 'supplemental_metadata.csv'\n    CHARMAP     = COMP_DIR / 'character_to_prediction_index.json'\n\n    # ── v5 Starting Checkpoint (Ensure this is the FINAL v5 model!) ─────────\n    # MUST point to the checkpoint that already has VOCAB_SIZE=62\n    CKPT_V5     = Path('/kaggle/input/models/tarunjoseph98/asl-transformer-v5-best/pytorch/default/1/asl_transformer_v5_best_progress update.pth')\n\n    # ── Outputs & Processed NPY Directories ─────────────────────────────────\n    OUTPUT_DIR  = Path('/kaggle/working')\n    NPY_TRAIN   = OUTPUT_DIR / 'npy_train'\n    NPY_SUPP    = OUTPUT_DIR / 'npy_supp'\n    \n    CKPT_BEST   = OUTPUT_DIR / 'asl_transformer_v6_best.pth'\n    CKPT_LAST   = OUTPUT_DIR / 'asl_transformer_v6_last.pth'\n\n    # ── Data & Model Dimensions ──────────────────────────────────────────────\n    FEATURE_SIZE   = 84     \n    MAX_SEQ_LEN    = 64\n    MAX_PHRASE_LEN = 34     \n    D_MODEL    = 384\n    ENC_LAYERS = 6\n    DEC_LAYERS = 6\n    N_HEADS    = 6\n    FFN_DIM    = 1024\n    EMBED_DIM  = 192\n    DROPOUT    = 0.25\n\n    # ── Training ──────────────────────────────────────────────────────────────\n    BATCH_SIZE     = 96\n    EPOCHS         = 30\n    LR             = 5e-5       \n    WEIGHT_DECAY   = 1e-4\n    WARMUP_EPOCHS  = 2\n    LABEL_SMOOTH   = 0.20\n    GRAD_CLIP      = 1.0\n    NUM_WORKERS    = 2\n    PATIENCE       = 10\n\n    # ── Scheduled Sampling ────────────────────────────────────────────────────\n    SS_START_EPOCH   = 8\n    SS_EPSILON_START = 1.0   \n    SS_EPSILON_END   = 0.3   \n\n    # ── Inference ─────────────────────────────────────────────────────────────\n    BEAM_WIDTH      = 5\n    LENGTH_PENALTY  = 0.6\n    RESUME_FROM     = None\n\n# Create working directories\nConfig.NPY_TRAIN.mkdir(parents=True, exist_ok=True)\nConfig.NPY_SUPP.mkdir(parents=True, exist_ok=True)\n\nfor p, tag in [(Config.TRAIN_META, 'TRAIN_META'), (Config.CHARMAP, 'CHARMAP'), \n               (Config.SUPP_META, 'SUPP_META'), (Config.CKPT_V5, 'CKPT_V5')]:\n    print(f\"{'✓' if p.exists() else '✗ MISSING':<10} {tag}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T02:48:23.30858Z","iopub.execute_input":"2026-03-23T02:48:23.309007Z","iopub.status.idle":"2026-03-23T02:48:23.322341Z","shell.execute_reply.started":"2026-03-23T02:48:23.308978Z","shell.execute_reply":"2026-03-23T02:48:23.321664Z"}},"outputs":[],"execution_count":null},{"id":"c19e47b8-4f1c-4c91-ad13-118651f5b848","cell_type":"markdown","source":"## 3. Vocabulary Definition","metadata":{}},{"id":"cell-vocab","cell_type":"code","source":"with open(Config.CHARMAP) as f:\n    char_to_idx = json.load(f)\n\nidx_to_char  = {v: k for k, v in char_to_idx.items()}\nN_CLASSES    = len(char_to_idx)         # 59\nSTART_IDX    = N_CLASSES                # 59\nEOS_IDX      = N_CLASSES + 1            # 60\nPAD_IDX      = N_CLASSES + 2            # 61\nVOCAB_SIZE   = N_CLASSES + 3            # 62\n\nprint(f'Real chars {N_CLASSES}  START={START_IDX}  EOS={EOS_IDX}  PAD={PAD_IDX}  VOCAB={VOCAB_SIZE}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T02:48:26.992673Z","iopub.execute_input":"2026-03-23T02:48:26.993234Z","iopub.status.idle":"2026-03-23T02:48:27.001341Z","shell.execute_reply.started":"2026-03-23T02:48:26.993204Z","shell.execute_reply":"2026-03-23T02:48:27.00077Z"}},"outputs":[],"execution_count":null},{"id":"0cd2190c-3ede-49a5-9c09-2cdc15340458","cell_type":"markdown","source":"## 4. Parquet to NPY Processing Logic (with Fixed Normalization)","metadata":{}},{"id":"68572506-95ba-4098-a214-f9cd4b08bc1f","cell_type":"code","source":"# FIX: added .any(axis=1, keepdims=True) back so we mask whole hands, not individual coordinates\ndef wrist_normalize(seq: np.ndarray) -> np.ndarray:\n    out = seq.copy()\n    for offset in [0, 42]:\n        lx = out[:, offset:offset+21]\n        ly = out[:, offset+21:offset+42]\n        wx, wy = lx[:, 0:1], ly[:, 0:1]\n        \n        visible = (lx != 0).any(axis=1, keepdims=True)  \n        \n        lx = np.where(visible, lx - wx, 0.0)\n        ly = np.where(visible, ly - wy, 0.0)\n        span = max(float(np.abs(lx).max()), float(np.abs(ly).max()), 1e-6)\n        out[:, offset:offset+21]  = lx / span\n        out[:, offset+21:offset+42] = ly / span\n    return out.astype(np.float32)\n\n_LH_X = [f'x_left_hand_{i}'  for i in range(21)]\n_LH_Y = [f'y_left_hand_{i}'  for i in range(21)]\n_RH_X = [f'x_right_hand_{i}' for i in range(21)]\n_RH_Y = [f'y_right_hand_{i}' for i in range(21)]\n_LOAD_COLS = ['frame'] + _LH_X + _LH_Y + _RH_X + _RH_Y\n\ndef process_parquets_to_npy(meta_df, base_dir, out_dir):\n    for file_id, group in tqdm(meta_df.groupby('file_id'), desc=f\"Processing to {out_dir.name}\"):\n        path = base_dir / f\"{file_id}.parquet\"\n        if not path.exists(): continue\n            \n        dfp = pd.read_parquet(path, columns=_LOAD_COLS)\n        seq_groups = dfp.groupby(dfp.index) \n        \n        for seq_id in group['sequence_id']:\n            if seq_id not in seq_groups.groups: continue\n                \n            rows = seq_groups.get_group(seq_id).sort_values('frame')\n            n = len(rows)\n            seq = np.zeros((n, 84), dtype=np.float32)\n            seq[:, 0:21]  = rows[_LH_X].values.astype(np.float32)\n            seq[:, 21:42] = rows[_LH_Y].values.astype(np.float32)\n            seq[:, 42:63] = rows[_RH_X].values.astype(np.float32)\n            seq[:, 63:84] = rows[_RH_Y].values.astype(np.float32)\n            seq = np.nan_to_num(seq, nan=0.0, posinf=0.0, neginf=0.0)\n            \n            seq = wrist_normalize(seq)\n            np.save(out_dir / f\"{seq_id}.npy\", seq)\n\ntrain_meta = pd.read_csv(Config.TRAIN_META)\nsupp_meta  = pd.read_csv(Config.SUPP_META)\n\nprint(f'Train sequences: {len(train_meta):,} | Supplemental sequences: {len(supp_meta):,}')\nprocess_parquets_to_npy(train_meta, Config.COMP_DIR / 'train_landmarks', Config.NPY_TRAIN)\nprocess_parquets_to_npy(supp_meta, Config.COMP_DIR / 'supplemental_landmarks', Config.NPY_SUPP)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T02:49:54.37269Z","iopub.execute_input":"2026-03-23T02:49:54.373175Z","iopub.status.idle":"2026-03-23T02:55:23.295053Z","shell.execute_reply.started":"2026-03-23T02:49:54.373144Z","shell.execute_reply":"2026-03-23T02:55:23.294181Z"}},"outputs":[],"execution_count":null},{"id":"4fe40d55-c5b7-499a-85be-a2e487959cbf","cell_type":"markdown","source":"## 5. Participant-Stratified Split (Main Data Only)","metadata":{}},{"id":"8aa83ad5-547f-4602-959a-5f4c8cce839d","cell_type":"code","source":"participants = train_meta['participant_id'].unique()\nnp.random.shuffle(participants)\nn = len(participants)\ntr_p  = set(participants[:int(n * 0.70)])\nval_p = set(participants[int(n * 0.70):int(n * 0.85)])\nte_p  = set(participants[int(n * 0.85):])\n\ntrain_split_meta = train_meta[train_meta['participant_id'].isin(tr_p)].reset_index(drop=True)\nval_split_meta   = train_meta[train_meta['participant_id'].isin(val_p)].reset_index(drop=True)\ntest_split_meta  = train_meta[train_meta['participant_id'].isin(te_p)].reset_index(drop=True)\n\nprint(f'Main Train: {len(train_split_meta):,} | Val: {len(val_split_meta):,} | Test: {len(test_split_meta):,}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T02:55:23.296666Z","iopub.execute_input":"2026-03-23T02:55:23.296902Z","iopub.status.idle":"2026-03-23T02:55:23.316012Z","shell.execute_reply.started":"2026-03-23T02:55:23.296878Z","shell.execute_reply":"2026-03-23T02:55:23.315493Z"}},"outputs":[],"execution_count":null},{"id":"79d7a435-6a22-4f00-bdda-71aa28f71032","cell_type":"markdown","source":"## 6. Augmentation & Datasets","metadata":{}},{"id":"74b6c911-bb4c-4e1a-b663-6299492b0538","cell_type":"code","source":"def _augment(seq: np.ndarray) -> np.ndarray:\n    n = len(seq)\n    \n    # 1. Random Frame Dropout\n    if random.random() < 0.5 and n > 8:                  \n        keep = sorted(random.sample(range(n), int(n * 0.9)))\n        seq  = seq[keep]\n        n = len(seq) # ✓ FIX: Update n so the new length is tracked\n        \n    # 2. Additive Gaussian Noise\n    if random.random() < 0.5:                              \n        seq += np.random.normal(0, 0.01, seq.shape).astype(np.float32)\n        \n    # 3. Spatial Flip (Left/Right Hand swap)\n    if random.random() < 0.5:                              \n        seq = np.concatenate([seq[:, 42:84], seq[:, 0:42]], axis=1)\n        seq[:, 0:21]  = -seq[:, 0:21]\n        seq[:, 42:63] = -seq[:, 42:63]\n        \n    # 4. Time Warping (Interpolation)\n    if random.random() < 0.4:                              \n        T_new = max(4, int(n * random.uniform(0.8, 1.2)))\n        seq   = seq[np.linspace(0, n-1, T_new, dtype=int)]\n        \n    return seq\n\ndef encode_phrase(phrase: str) -> list:\n    chars = [char_to_idx.get(c, PAD_IDX) for c in phrase[:Config.MAX_PHRASE_LEN - 2]]\n    seq   = [START_IDX] + chars + [EOS_IDX]\n    seq  += [PAD_IDX] * (Config.MAX_PHRASE_LEN - len(seq))\n    return seq[:Config.MAX_PHRASE_LEN]\n\nclass ProcessedNPYDataset(Dataset):\n    def __init__(self, df, npy_dir, augment=False):\n        self.df = df.reset_index(drop=True)\n        self.npy_dir = npy_dir\n        self.augment = augment\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, i):\n        row = self.df.iloc[i]\n        path = self.npy_dir / f\"{row['sequence_id']}.npy\"\n        \n        if path.exists():\n            seq = np.load(path).astype(np.float32)\n        else:\n            seq = np.zeros((Config.MAX_SEQ_LEN, 84), dtype=np.float32)\n            \n        if self.augment:\n            seq = _augment(seq)\n\n        T = len(seq)\n        if T > Config.MAX_SEQ_LEN:\n            seq = seq[np.linspace(0, T-1, Config.MAX_SEQ_LEN, dtype=int)]\n        elif T < Config.MAX_SEQ_LEN:\n            seq = np.vstack([seq, np.zeros((Config.MAX_SEQ_LEN - T, 84), dtype=np.float32)])\n\n        x = torch.from_numpy(seq.T).float()\n        y = torch.tensor(encode_phrase(str(row['phrase'])), dtype=torch.long)\n        return x, y\n\nds_main_train = ProcessedNPYDataset(train_split_meta, Config.NPY_TRAIN, augment=True)\nds_supp_train = ProcessedNPYDataset(supp_meta, Config.NPY_SUPP, augment=True)\ncombined_train_ds = ConcatDataset([ds_main_train, ds_supp_train])\nval_ds  = ProcessedNPYDataset(val_split_meta, Config.NPY_TRAIN, augment=False)\n\ndef make_loader(ds, shuffle):\n    return DataLoader(ds, batch_size=Config.BATCH_SIZE, shuffle=shuffle,\n                      num_workers=Config.NUM_WORKERS, pin_memory=True,\n                      persistent_workers=Config.NUM_WORKERS > 0)\n\ntrain_loader = make_loader(combined_train_ds, shuffle=True)\nval_loader   = make_loader(val_ds, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T02:55:23.316827Z","iopub.execute_input":"2026-03-23T02:55:23.317034Z","iopub.status.idle":"2026-03-23T02:55:23.331594Z","shell.execute_reply.started":"2026-03-23T02:55:23.317014Z","shell.execute_reply":"2026-03-23T02:55:23.330981Z"}},"outputs":[],"execution_count":null},{"id":"35e12dbb-573b-4c8c-85e6-9c2f0cb4964d","cell_type":"markdown","source":"## 7. Conformer-Transformer Model Definition","metadata":{}},{"id":"37b67821-084b-4898-a913-63ae5131960f","cell_type":"code","source":"class PositionalEncoding(nn.Module):\n    def __init__(self, d_model, maxlen=512, dropout=0.1):\n        super().__init__()\n        self.drop = nn.Dropout(dropout)\n        pe  = torch.zeros(maxlen, d_model)\n        pos = torch.arange(maxlen).unsqueeze(1).float()\n        div = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(pos * div)\n        pe[:, 1::2] = torch.cos(pos * div)\n        self.register_buffer('pe', pe.unsqueeze(0))\n\n    def forward(self, x):\n        return self.drop(x + self.pe[:, :x.size(1)])\n\nclass ConformerBlock(nn.Module):\n    def __init__(self, d_model, n_heads, ffn_dim, kernel_size=31, dropout=0.1):\n        super().__init__()\n        self.ff1       = nn.Sequential(nn.LayerNorm(d_model), nn.Linear(d_model, ffn_dim), nn.SiLU(), nn.Dropout(dropout), nn.Linear(ffn_dim, d_model), nn.Dropout(dropout))\n        self.norm_attn = nn.LayerNorm(d_model)\n        self.attn      = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True)\n        self.drop_attn = nn.Dropout(dropout)\n        self.norm_conv = nn.LayerNorm(d_model)\n        self.conv      = nn.Sequential(nn.Conv1d(d_model, 2*d_model, 1), nn.GLU(dim=1), nn.Conv1d(d_model, d_model, kernel_size, padding=kernel_size//2, groups=d_model), nn.BatchNorm1d(d_model), nn.SiLU(), nn.Conv1d(d_model, d_model, 1), nn.Dropout(dropout))\n        self.ff2      = nn.Sequential(nn.LayerNorm(d_model), nn.Linear(d_model, ffn_dim), nn.SiLU(), nn.Dropout(dropout), nn.Linear(ffn_dim, d_model), nn.Dropout(dropout))\n        self.norm_out = nn.LayerNorm(d_model)\n\n    def forward(self, x):\n        x = x + 0.5 * self.ff1(x)\n        r, _ = self.attn(self.norm_attn(x), self.norm_attn(x), self.norm_attn(x))\n        x = x + self.drop_attn(r)\n        x = x + self.conv(self.norm_conv(x).transpose(1, 2)).transpose(1, 2)\n        x = x + 0.5 * self.ff2(x)\n        return self.norm_out(x)\n\nclass ConformerEncoder(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.proj   = nn.Sequential(nn.Conv1d(cfg.FEATURE_SIZE, cfg.D_MODEL, kernel_size=3, padding=1), nn.BatchNorm1d(cfg.D_MODEL), nn.ReLU())\n        self.posenc = PositionalEncoding(cfg.D_MODEL, dropout=cfg.DROPOUT)\n        self.layers = nn.ModuleList([ConformerBlock(cfg.D_MODEL, cfg.N_HEADS, cfg.FFN_DIM, dropout=cfg.DROPOUT) for _ in range(cfg.ENC_LAYERS)])\n\n    def forward(self, x):\n        x = self.proj(x).permute(0, 2, 1)\n        x = self.posenc(x)\n        for layer in self.layers:\n            x = layer(x)\n        return x \n\nclass TransformerDecoder(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.embed    = nn.Embedding(VOCAB_SIZE, cfg.EMBED_DIM, padding_idx=PAD_IDX)\n        self.proj_emb = nn.Linear(cfg.EMBED_DIM, cfg.D_MODEL)\n        self.posenc   = PositionalEncoding(cfg.D_MODEL, dropout=cfg.DROPOUT)\n        dec_layer     = nn.TransformerDecoderLayer(d_model=cfg.D_MODEL, nhead=cfg.N_HEADS, dim_feedforward=cfg.FFN_DIM, dropout=cfg.DROPOUT, batch_first=True, norm_first=True)\n        self.decoder  = nn.TransformerDecoder(dec_layer, num_layers=cfg.DEC_LAYERS)\n        self.fc_out   = nn.Linear(cfg.D_MODEL, VOCAB_SIZE)\n\n    def forward(self, tgt, memory):\n        L          = tgt.size(1)\n        tgt_mask   = nn.Transformer.generate_square_subsequent_mask(L, device=tgt.device)\n        tgt_kp_mask = (tgt == PAD_IDX)\n        emb        = self.posenc(self.proj_emb(self.embed(tgt)))\n        out        = self.decoder(emb, memory, tgt_mask=tgt_mask, tgt_key_padding_mask=tgt_kp_mask)\n        return self.fc_out(out)\n\nclass ASLConformerSeq2Seq(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.encoder = ConformerEncoder(cfg)\n        self.decoder = TransformerDecoder(cfg)\n\n    def forward(self, x, tgt):\n        return self.decoder(tgt, self.encoder(x))\n\nmodel = ASLConformerSeq2Seq(Config).to(DEVICE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T02:55:23.333061Z","iopub.execute_input":"2026-03-23T02:55:23.333266Z","iopub.status.idle":"2026-03-23T02:55:23.919226Z","shell.execute_reply.started":"2026-03-23T02:55:23.333238Z","shell.execute_reply":"2026-03-23T02:55:23.918654Z"}},"outputs":[],"execution_count":null},{"id":"a7963bfb-2b42-40f9-b060-715318c8c719","cell_type":"markdown","source":"## 8. Load v5 Checkpoint","metadata":{}},{"id":"b4076324-8c84-4ffe-8442-2fda7e89a278","cell_type":"code","source":"if Config.CKPT_V5.exists():\n    ckpt_v5  = torch.load(Config.CKPT_V5, map_location=DEVICE, weights_only=False)\n    v5_state = ckpt_v5.get('model_state_dict', ckpt_v5)\n    v6_state = model.state_dict()\n\n    transferred, skipped, surgeried = 0, 0, 0\n    \n    for name, param in v5_state.items():\n        if name in v6_state:\n            # 1. Exact shape match\n            if v6_state[name].shape == param.shape:\n                v6_state[name].copy_(param)\n                transferred += 1\n            \n            # 2. Shape mismatch (Surgery for Vocab Size changes)\n            elif 'fc_out' in name or 'embed' in name:\n                min_dim_0 = min(param.shape[0], v6_state[name].shape[0])\n                \n                if len(param.shape) == 1:  # Bias tensor (1D)\n                    v6_state[name][:min_dim_0].copy_(param[:min_dim_0])\n                else:                      # Weight tensor (2D)\n                    min_dim_1 = min(param.shape[1], v6_state[name].shape[1])\n                    v6_state[name][:min_dim_0, :min_dim_1].copy_(param[:min_dim_0, :min_dim_1])\n                \n                print(f\"⚠️ Surgeried '{name}': Resized from {list(param.shape)} to {list(v6_state[name].shape)}\")\n                surgeried += 1\n                transferred += 1\n            \n            # 3. Other unexplained mismatches\n            else:\n                skipped += 1\n\n    model.load_state_dict(v6_state)\n    print(f'\\nStats: {transferred} transferred (including {surgeried} resized), {skipped} skipped.')\n    \n    if skipped > 0:\n        print(f'❌ WARNING: Skipped {skipped} tensors for reasons other than vocab size.')\n    else:\n        print('✓ Checkpoint loaded successfully! You are safe to proceed.')\nelse:\n    print(f'❌ Checkpoint missing at {Config.CKPT_V5}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T02:55:23.920111Z","iopub.execute_input":"2026-03-23T02:55:23.920345Z","iopub.status.idle":"2026-03-23T02:55:25.359928Z","shell.execute_reply.started":"2026-03-23T02:55:23.920321Z","shell.execute_reply":"2026-03-23T02:55:25.35929Z"}},"outputs":[],"execution_count":null},{"id":"5a4d2ff1-5d80-4819-a278-c31126d65c88","cell_type":"markdown","source":"## 9. Sanity Check: Beam CER on Val","metadata":{}},{"id":"c39de17c-feb4-4ed9-a9e1-eae6caeefce5","cell_type":"code","source":"def decode_seq(indices):\n    chars = []\n    for i in indices:\n        if i == EOS_IDX: break\n        if i not in (PAD_IDX, START_IDX) and i in idx_to_char:\n            chars.append(idx_to_char[i])\n    return ''.join(chars)\n\n@torch.no_grad()\ndef beam_search(model, x, beam_width=5, max_len=34, length_penalty=0.6):\n    model.eval()\n    memory = model.encoder(x.unsqueeze(0).to(DEVICE))\n    beams  = [([START_IDX], 0.0)]\n    for _ in range(max_len - 1):\n        candidates = []\n        for seq, score in beams:\n            if seq[-1] == EOS_IDX:\n                candidates.append((seq, score))\n                continue\n            tgt    = torch.tensor([seq], dtype=torch.long, device=DEVICE)\n            logits = model.decoder(tgt, memory)\n            logp   = F.log_softmax(logits[0, -1], dim=-1)\n            topk   = logp.topk(beam_width)\n            for lp, idx in zip(topk.values, topk.indices):\n                candidates.append((seq + [idx.item()], score + lp.item()))\n        beams = sorted(candidates, key=lambda b: b[1] / max(len(b[0]) - 1, 1) ** length_penalty, reverse=True)[:beam_width]\n        if all(b[0][-1] == EOS_IDX for b in beams): break\n    return beams[0][0]\n\nmodel.eval()\ncer_vals = []\n\npbar = tqdm(range(200), desc=\"Sanity beam CER\")\nfor i in pbar:\n    x, y = val_ds[i]\n    pred = beam_search(model, x, Config.BEAM_WIDTH, Config.MAX_PHRASE_LEN, Config.LENGTH_PENALTY)\n    ps, ts = decode_seq(pred), decode_seq(y.numpy())\n    if len(ts) > 0:\n        cer_vals.append(editdistance.eval(ps, ts) / len(ts))\n    if (i + 1) % 50 == 0:\n        pbar.set_postfix({'CER': f'{np.mean(cer_vals):.4f}'})\n\nprint(f'Sanity beam CER: {np.mean(cer_vals):.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T02:55:25.360889Z","iopub.execute_input":"2026-03-23T02:55:25.36118Z","iopub.status.idle":"2026-03-23T02:57:27.980694Z","shell.execute_reply.started":"2026-03-23T02:55:25.361149Z","shell.execute_reply":"2026-03-23T02:57:27.979796Z"}},"outputs":[],"execution_count":null},{"id":"51240fe9-35ef-46b2-9989-3caed1a75920","cell_type":"markdown","source":"## 10. Advanced Training Setup & Functions","metadata":{}},{"id":"aa30501c-0c2a-477e-ba83-bb19ad2c527f","cell_type":"code","source":"criterion = nn.CrossEntropyLoss(ignore_index=PAD_IDX, label_smoothing=Config.LABEL_SMOOTH)\noptimizer = optim.AdamW(model.parameters(), lr=Config.LR, weight_decay=Config.WEIGHT_DECAY)\nscaler    = GradScaler('cuda')\n\ndef char_acc(logits, targets):\n    preds = logits.argmax(-1)\n    mask  = (targets != PAD_IDX)\n    return (preds[mask] == targets[mask]).float().mean().item()\n\ndef train_one_epoch():\n    model.train()\n    tot_loss = tot_acc = n = 0\n    for x, y in tqdm(train_loader, desc='train', leave=False):\n        x, y = x.to(DEVICE), y.to(DEVICE)\n        optimizer.zero_grad(set_to_none=True)\n        \n        with autocast('cuda', enabled=True):\n            logits = model(x, y[:, :-1])\n            target = y[:, 1:].contiguous()\n            loss   = criterion(logits.reshape(-1, VOCAB_SIZE), target.reshape(-1))\n            \n        scaler.scale(loss).backward()\n        scaler.unscale_(optimizer)\n        nn.utils.clip_grad_norm_(model.parameters(), Config.GRAD_CLIP)\n        \n        scaler.step(optimizer); scaler.update()\n        \n        tot_loss += loss.item(); tot_acc += char_acc(logits.float(), target); n += 1\n    return tot_loss / n, tot_acc / n\n\n@torch.no_grad()\ndef validate():\n    model.eval()\n    tot_loss = tot_acc = n = 0\n    all_preds, all_tgts = [], []\n    for x, y in tqdm(val_loader, desc='val', leave=False):\n        x, y = x.to(DEVICE), y.to(DEVICE)\n        \n        with autocast('cuda', enabled=True):\n            logits = model(x, y[:, :-1])\n            target = y[:, 1:].contiguous()\n            loss   = criterion(logits.reshape(-1, VOCAB_SIZE), target.reshape(-1))\n            \n        tot_loss += loss.item(); tot_acc += char_acc(logits.float(), target)\n        all_preds.extend(logits.argmax(-1).cpu().numpy()); all_tgts.extend(target.cpu().numpy()); n += 1\n        \n    cer_vals = [editdistance.eval(decode_seq(p), decode_seq(t)) / max(len(decode_seq(t)), 1)\n                for p, t in zip(all_preds, all_tgts)]\n    return tot_loss / n, tot_acc / n, float(np.mean(cer_vals))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T03:12:10.095422Z","iopub.execute_input":"2026-03-23T03:12:10.096471Z","iopub.status.idle":"2026-03-23T03:12:12.551476Z","shell.execute_reply.started":"2026-03-23T03:12:10.096411Z","shell.execute_reply":"2026-03-23T03:12:12.550737Z"}},"outputs":[],"execution_count":null},{"id":"4437dec2-dc1f-4ee5-9b71-d26dc40bfb5e","cell_type":"markdown","source":"## 11. Main Execution Loop","metadata":{}},{"id":"0b7ee4f1-8262-496e-8abf-7ef98cdc1520","cell_type":"code","source":"# Initialize history dictionary\nhistory = {k: [] for k in ('tr_loss','tr_acc','vl_loss','vl_acc','cer','epsilon')}\n\nbest_cer = float('inf')\npatience_ctr = 0\n\nprint(\"Starting Fast Regularization Loop (Tracking Best CER & History)...\")\nfor epoch in range(1, Config.EPOCHS + 1):\n    t0 = time.time()\n    \n    # Epsilon is always 1.0 because we are using 100% Teacher Forcing for speed\n    epsilon = 1.0 \n    \n    tr_loss, tr_acc = train_one_epoch()\n    vl_loss, vl_acc, cer = validate()\n    \n    # Save metrics to history for plotting in Cell 12\n    for k, v in zip(('tr_loss','tr_acc','vl_loss','vl_acc','cer','epsilon'), \n                    (tr_loss, tr_acc, vl_loss, vl_acc, cer, epsilon)):\n        history[k].append(v)\n\n    tag = ''\n    # Tracking CER to grab the weights that actually spell the best\n    if cer < best_cer:\n        best_cer = cer\n        patience_ctr = 0\n        tag = ' ← best'\n        torch.save({'model_state_dict': model.state_dict(), 'best_cer': best_cer}, Config.CKPT_BEST)\n    else:\n        patience_ctr += 1\n        tag = f' patience {patience_ctr}/{Config.PATIENCE}'\n\n    print(f'Epoch {epoch:02d}/{Config.EPOCHS} | tr {tr_loss:.4f}/{tr_acc:.4f} | vl {vl_loss:.4f}/{vl_acc:.4f} | CER {cer:.4f} | {time.time() - t0:.0f}s{tag}')\n    \n    if patience_ctr >= Config.PATIENCE:\n        print('Early stopping triggered. Training complete.')\n        break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T03:12:14.553473Z","iopub.execute_input":"2026-03-23T03:12:14.55421Z","iopub.status.idle":"2026-03-23T03:46:29.363846Z","shell.execute_reply.started":"2026-03-23T03:12:14.55418Z","shell.execute_reply":"2026-03-23T03:46:29.363001Z"}},"outputs":[],"execution_count":null},{"id":"d2485668-75d5-41c9-bf34-fe4592260e8b","cell_type":"markdown","source":"## 12. Training Curves Visualization","metadata":{}},{"id":"3accc658-250c-40b2-a96a-9832bef81357","cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# 1. Manually reconstruct the history dictionary from your printed logs\nhistory = {\n    'tr_loss': [2.0400, 1.8847, 1.8159, 1.7682, 1.7335, 1.7076, 1.6860, 1.6697, 1.6561, 1.6446, 1.6347],\n    'tr_acc':  [0.7186, 0.7790, 0.8086, 0.8295, 0.8446, 0.8554, 0.8640, 0.8701, 0.8749, 0.8788, 0.8820],\n    'vl_loss': [1.8281, 1.8361, 1.8433, 1.8503, 1.8536, 1.8554, 1.8600, 1.8592, 1.8625, 1.8621, 1.8601],\n    'vl_acc':  [0.8020, 0.7953, 0.7908, 0.7868, 0.7842, 0.7834, 0.7812, 0.7812, 0.7789, 0.7790, 0.7789],\n    'cer':     [0.2385, 0.2476, 0.2457, 0.2529, 0.2567, 0.2549, 0.2570, 0.2546, 0.2584, 0.2509, 0.2547],\n    'epsilon': [1.0] * 11  # Epsilon was 1.0 for all epochs in the fast loop\n}\n\n# 2. Plot Training Curves\nfig, axes = plt.subplots(1, 3, figsize=(18, 5))\nepochs_range = range(1, len(history['tr_loss']) + 1)\n\n# Loss Plot\naxes[0].plot(epochs_range, history['tr_loss'], label='Train Loss', color='blue')\naxes[0].plot(epochs_range, history['vl_loss'], label='Val Loss', color='orange')\naxes[0].set_title('Cross Entropy Loss')\naxes[0].set_xlabel('Epoch'); axes[0].set_ylabel('Loss')\naxes[0].legend(); axes[0].grid(True, linestyle='--', alpha=0.6)\n\n# Accuracy Plot\naxes[1].plot(epochs_range, history['tr_acc'], label='Train Acc', color='blue')\naxes[1].plot(epochs_range, history['vl_acc'], label='Val Acc', color='orange')\naxes[1].set_title('Teacher-Forced Token Accuracy')\naxes[1].set_xlabel('Epoch'); axes[1].set_ylabel('Accuracy')\naxes[1].legend(); axes[1].grid(True, linestyle='--', alpha=0.6)\n\n# CER & Epsilon Plot\nax3 = axes[2]\nax3.plot(epochs_range, history['cer'], label='Val CER (Beam)', color='green', linewidth=2)\nax3.set_title('Character Error Rate & SS Epsilon')\nax3.set_xlabel('Epoch'); ax3.set_ylabel('CER', color='green')\nax3.tick_params(axis='y', labelcolor='green')\nax3.grid(True, linestyle='--', alpha=0.6)\n\nax3_twin = ax3.twinx()\nax3_twin.plot(epochs_range, history['epsilon'], label='SS Epsilon', color='red', linestyle=':')\nax3_twin.set_ylabel('Scheduled Sampling ε', color='red')\nax3_twin.tick_params(axis='y', labelcolor='red')\nax3_twin.set_ylim([0, 1.1]) # Lock the y-axis for epsilon so the flat line looks correct\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T06:38:21.969408Z","iopub.execute_input":"2026-03-23T06:38:21.969764Z","iopub.status.idle":"2026-03-23T06:38:22.498103Z","shell.execute_reply.started":"2026-03-23T06:38:21.969737Z","shell.execute_reply":"2026-03-23T06:38:22.497497Z"}},"outputs":[],"execution_count":null},{"id":"5ba31d6d-7811-44a1-9c0c-0250d21e64a6","cell_type":"markdown","source":"## 13. Load Best Weights & Prepare Test Data","metadata":{}},{"id":"b2c7946d-8b62-4c54-94d2-a47c64829111","cell_type":"code","source":"# 1. Define the missing test dataset (My apologies for dropping this!)\ntest_ds = ProcessedNPYDataset(test_split_meta, Config.NPY_TRAIN, augment=False)\n\n# 2. Load the best weights from the training loop\nbest_ckpt = torch.load(Config.CKPT_BEST, map_location=DEVICE, weights_only=True)\nmodel.load_state_dict(best_ckpt['model_state_dict'])\nmodel.eval()\nprint(\"✓ Best validation checkpoint loaded for evaluation.\")\n\n# 3. Create the test dataloader\ntest_loader = make_loader(test_ds, shuffle=False)\nprint(f\"Test Set Size: {len(test_ds):,} sequences\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T04:12:50.880629Z","iopub.execute_input":"2026-03-23T04:12:50.880938Z","iopub.status.idle":"2026-03-23T04:12:51.021804Z","shell.execute_reply.started":"2026-03-23T04:12:50.880912Z","shell.execute_reply":"2026-03-23T04:12:51.021164Z"}},"outputs":[],"execution_count":null},{"id":"01928735-29ae-4198-9037-523f79331435","cell_type":"markdown","source":"## 14. Optional — Beam Width & Length Penalty Sweep\n\nThis runs a fast grid search over 200 validation samples to find the optimal decoding parameters before hitting the test set.","metadata":{}},{"id":"aabd8df2-bfc9-464d-95b0-4349a627d8b5","cell_type":"code","source":"@torch.no_grad()\ndef sweep_beam_params(model, dataset, n_samples=200):\n    model.eval()\n    beam_widths = [1, 3, 5]\n    length_penalties = [0.6, 0.8, 1.0]\n    \n    results = {}\n    print(\"Starting Beam Search Hyperparameter Sweep...\")\n    \n    for bw in beam_widths:\n        for lp in length_penalties:\n            cer_vals = []\n            # Use a fixed subset for fair comparison\n            for i in range(min(n_samples, len(dataset))):\n                x, y = dataset[i]\n                pred = beam_search(model, x, beam_width=bw, max_len=Config.MAX_PHRASE_LEN, length_penalty=lp)\n                ps, ts = decode_seq(pred), decode_seq(y.numpy())\n                if len(ts) > 0:\n                    cer_vals.append(editdistance.eval(ps, ts) / len(ts))\n            \n            avg_cer = np.mean(cer_vals)\n            results[(bw, lp)] = avg_cer\n            print(f\"Beam Width: {bw} | Length Penalty: {lp:.1f}  =>  CER: {avg_cer:.4f}\")\n            \n    best_params = min(results, key=results.get)\n    print(f\"\\n🏆 Best Params -> Beam Width: {best_params[0]}, Length Penalty: {best_params[1]} (CER: {results[best_params]:.4f})\")\n    return best_params\n\n# Run sweep (Comment out if you just want to use Config defaults)\nbest_bw, best_lp = sweep_beam_params(model, val_ds, n_samples=200)\n\n# Update Config with best parameters found\nConfig.BEAM_WIDTH = best_bw\nConfig.LENGTH_PENALTY = best_lp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T03:59:27.935829Z","iopub.execute_input":"2026-03-23T03:59:27.936116Z","iopub.status.idle":"2026-03-23T04:10:50.33411Z","shell.execute_reply.started":"2026-03-23T03:59:27.93609Z","shell.execute_reply":"2026-03-23T04:10:50.333403Z"}},"outputs":[],"execution_count":null},{"id":"d8ce1b0f-894b-4e60-a698-ac59850b627e","cell_type":"markdown","source":"## 15. Final Evaluation — Held-Out Test Set","metadata":{}},{"id":"385b6daa-214d-43b2-8dcf-ffd19a7e77bf","cell_type":"code","source":"@torch.no_grad()\ndef evaluate_test_set(model, dataset, bw, lp):\n    model.eval()\n    cer_vals = []\n    test_preds, test_tgts = [], []\n    \n    pbar = tqdm(range(len(dataset)), desc=\"Testing Model\")\n    for i in pbar:\n        x, y = dataset[i]\n        pred = beam_search(model, x, beam_width=bw, max_len=Config.MAX_PHRASE_LEN, length_penalty=lp)\n        ps, ts = decode_seq(pred), decode_seq(y.numpy())\n        \n        test_preds.append(ps)\n        test_tgts.append(ts)\n        \n        if len(ts) > 0:\n            cer = editdistance.eval(ps, ts) / len(ts)\n            cer_vals.append(cer)\n            \n        if (i + 1) % 50 == 0:\n            pbar.set_postfix({'Test CER': f'{np.mean(cer_vals):.4f}'})\n            \n    return np.mean(cer_vals), test_preds, test_tgts\n\nfinal_cer, preds, tgts = evaluate_test_set(model, test_ds, Config.BEAM_WIDTH, Config.LENGTH_PENALTY)\n\nprint(f\"\\n{'='*40}\")\nprint(f\"FINAL TEST CER: {final_cer:.4f}\")\nprint(f\"{'='*40}\\n\")\n\nprint(\"Sample Predictions:\")\nprint(f\"{'Match':<6} | {'Ground Truth':<30} | {'Prediction'}\")\nprint(\"-\" * 70)\nfor gt, pred in zip(tgts[:20], preds[:20]):\n    match = '✓' if gt == pred else '✗'\n    print(f\"{match:<6} | {gt:<30} | {pred}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T04:25:28.306214Z","iopub.execute_input":"2026-03-23T04:25:28.306529Z","iopub.status.idle":"2026-03-23T06:14:20.642614Z","shell.execute_reply.started":"2026-03-23T04:25:28.306505Z","shell.execute_reply":"2026-03-23T06:14:20.641823Z"}},"outputs":[],"execution_count":null},{"id":"4901de0c-658a-4452-99de-668d0666fdb6","cell_type":"markdown","source":"## 16. Save Final Checkpoint (Packaged for Deployment)\n\nThis builds a \"fat\" checkpoint dictionary containing everything needed to reconstruct the model without hardcoding hyperparameters in the inference scripts.","metadata":{}},{"id":"9c439b1c-1f0a-4ab2-bc98-648543366f9f","cell_type":"code","source":"final_ckpt_path = Config.OUTPUT_DIR / 'asl_transformer_v6_final.pth'\n\ntorch.save({\n    'model_state_dict': model.state_dict(),\n    'char_to_idx':      char_to_idx,\n    'idx_to_char':      idx_to_char,\n    'feature_size':     Config.FEATURE_SIZE,\n    'vocab_size':       VOCAB_SIZE,\n    'n_classes':        N_CLASSES,\n    'start_idx':        START_IDX,\n    'eos_idx':          EOS_IDX,\n    'pad_idx':          PAD_IDX,\n    'max_seq_len':      Config.MAX_SEQ_LEN,\n    'max_phrase_len':   Config.MAX_PHRASE_LEN,\n    'd_model':          Config.D_MODEL,\n    'enc_layers':       Config.ENC_LAYERS,\n    'dec_layers':       Config.DEC_LAYERS,\n    'n_heads':          Config.N_HEADS,\n    'ffn_dim':          Config.FFN_DIM,\n    'embed_dim':        Config.EMBED_DIM,\n    'test_cer':         final_cer,\n    'beam_width':       Config.BEAM_WIDTH,\n    'length_penalty':   Config.LENGTH_PENALTY\n}, final_ckpt_path)\n\nprint(f\"Final model architecture, vocabulary, and weights successfully saved to:\\n{final_ckpt_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T06:14:20.643903Z","iopub.execute_input":"2026-03-23T06:14:20.644174Z","iopub.status.idle":"2026-03-23T06:14:20.797739Z","shell.execute_reply.started":"2026-03-23T06:14:20.644144Z","shell.execute_reply":"2026-03-23T06:14:20.797054Z"}},"outputs":[],"execution_count":null},{"id":"421825df-6b8f-47fc-be05-83442ce89e77","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}