{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":52950,"databundleVersionId":5973250}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 🏗️ Update: Try new architectures (ST-GCN)\n\n### Changes in this version:\n- Reshape the Input Data\n- Define MediaPipe Hand Graph\n- Implement ST-GCN Architecture\n- Update Model Instantiation and W&B Logging\n- Added CER evaluation on training\n\n## ✋🏽 Update: Landmarks Normalization\n\n### Changes in this version:\n- **Integrate: Landmarks position normalization**\n- (https://www.kaggle.com/code/lejuin/fingerspelling-data-prep#Center-hand)\n\n## 📈 Update: W&B\n\n### Changes in this version:\n- **Integrate: Metrics monitoring in W&B team workspace**\n- (https://wandb.ai/inaki-rodriguez-reyes-upc-universidad-peruana-de-ciencia/asl-fingerspelling-previews)\n\n## 🔡 Update: Words\n\n### Changes in this version:\n- **Integrate: Filter only words with a-z**\n- (https://www.kaggle.com/code/pauerv/notebook136616d653-v12-overfit)\n- **Reduced dataset size**\n  - Train: 6564\n  - Val: 863\n  - Test: 920","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2026-03-05T09:14:45.78153Z","iopub.execute_input":"2026-03-05T09:14:45.782457Z","iopub.status.idle":"2026-03-05T09:14:47.534641Z","shell.execute_reply.started":"2026-03-05T09:14:45.782423Z","shell.execute_reply":"2026-03-05T09:14:47.533508Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Install and configure W&B on this notebook\n!pip install wandb -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:14:47.536607Z","iopub.execute_input":"2026-03-05T09:14:47.537097Z","iopub.status.idle":"2026-03-05T09:14:52.941075Z","shell.execute_reply.started":"2026-03-05T09:14:47.537068Z","shell.execute_reply":"2026-03-05T09:14:52.939747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfrom kaggle_secrets import UserSecretsClient\nimport wandb\n\nuser_secrets = UserSecretsClient()\nsecret_value = user_secrets.get_secret(\"wandb\")\nos.environ[\"WANDB_API_KEY\"] = secret_value\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:14:52.942833Z","iopub.execute_input":"2026-03-05T09:14:52.943297Z","iopub.status.idle":"2026-03-05T09:14:56.783856Z","shell.execute_reply.started":"2026-03-05T09:14:52.943259Z","shell.execute_reply":"2026-03-05T09:14:56.783059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport json\n\n# Lectura de parquets \nimport pyarrow.parquet as pq\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\n\nSEED = 42\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:14:56.784906Z","iopub.execute_input":"2026-03-05T09:14:56.785329Z","iopub.status.idle":"2026-03-05T09:15:01.86149Z","shell.execute_reply.started":"2026-03-05T09:14:56.7853Z","shell.execute_reply":"2026-03-05T09:15:01.860508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRAIN_CSV_PATH = \"/kaggle/input/asl-fingerspelling/train.csv\"\n\ntrain_df = pd.read_csv(TRAIN_CSV_PATH)\n\nprint(\"Total muestras:\", len(train_df))\ntrain_df.head()\n\n## para que entendamos que hay dentro de los parquets, conjunto de líneas que representan un frame identificado a la vedz por un sequence_id.\n## como veremos en cada línea del parquet hay muchas coordenadas, de cara, mano izqueirda, mano derecha, etc, a nosotros solo nos va a interesar\n#coordenadas de la mano derecha que supuestamente es la que va a representar el lenguaje de signos\n\n## el dataset se divide en parquets por temas de eficiencia y porque sería poco eficiente almacenar todo en un solo parquet, las consultas \n## a las diferentes secuencias serían muy lentas\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:01.864072Z","iopub.execute_input":"2026-03-05T09:15:01.864733Z","iopub.status.idle":"2026-03-05T09:15:02.058093Z","shell.execute_reply.started":"2026-03-05T09:15:01.864701Z","shell.execute_reply":"2026-03-05T09:15:02.057084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Vamos a hacer una función que filtre el \"phrase\" del dataset para que recibamos una única palabra i mediante una expresión regultar\n#que nos saque solo palabras que contengas letras del abecedario, sin números ni carácteres raros.\n\nimport re\n\ndef is_single_word(word):\n    return bool(re.fullmatch(r\"[A-Za-z]+\", word))\n\ntrain_df[\"is_single_word\"] = train_df[\"phrase\"].apply(is_single_word)\n\nsingle_word_df = train_df[train_df[\"is_single_word\"]].copy()\n\nprint(\"Total muestras:\", len(train_df))\nprint(\"Solo palabras:\", len(single_word_df))\nprint(\"Porcentaje:\", len(single_word_df) / len(train_df) * 100)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.059344Z","iopub.execute_input":"2026-03-05T09:15:02.059781Z","iopub.status.idle":"2026-03-05T09:15:02.135822Z","shell.execute_reply.started":"2026-03-05T09:15:02.059743Z","shell.execute_reply":"2026-03-05T09:15:02.134978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#No hay suficientes palabras sueltas para considerar únicament las muestras de una palabra. Como vemos, una única palabra son solo el 1% del dataset\n# y seguramente de estas palabras sueltas algunas contendran caràcteres especiales y números que queremos excluir, por lo que no usaremos este método\n\n#Como lo que queremos es detectar letra a letra, creemos que no es crítico\n\ntrain_df[\"phrase\"].head(20)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.1371Z","iopub.execute_input":"2026-03-05T09:15:02.137498Z","iopub.status.idle":"2026-03-05T09:15:02.145556Z","shell.execute_reply.started":"2026-03-05T09:15:02.137451Z","shell.execute_reply":"2026-03-05T09:15:02.144485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Como vemos, aparecen muchos carácteres especiales o números, deberíamos limpiar estas entradas.\n\nfrom collections import Counter\nimport string\n\nletters = set(string.ascii_lowercase)\n\nletter_counter = Counter()\n\nfor phrase in train_df[\"phrase\"].astype(str):\n    for ch in phrase.lower():\n        if ch in letters:\n            letter_counter[ch] += 1\n\nletter_counter","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.146826Z","iopub.execute_input":"2026-03-05T09:15:02.14725Z","iopub.status.idle":"2026-03-05T09:15:02.429325Z","shell.execute_reply.started":"2026-03-05T09:15:02.147183Z","shell.execute_reply":"2026-03-05T09:15:02.428378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(letter_counter), letter_counter.most_common()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.430555Z","iopub.execute_input":"2026-03-05T09:15:02.431017Z","iopub.status.idle":"2026-03-05T09:15:02.43788Z","shell.execute_reply.started":"2026-03-05T09:15:02.430979Z","shell.execute_reply":"2026-03-05T09:15:02.436765Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Podemos ver aquí que tenemos muestras que incluyen las 26 letras del abecedario y con bastantes apariciones, por ejemplo la letra e aparece en 71986 veces\n# o la que menos, la q con 1114.\nimport re\n\ndef is_clean_phrase(phrase):\n    return bool(re.fullmatch(r\"[A-Za-z ]+\", phrase))\n\nclean_df = train_df[train_df[\"phrase\"].apply(is_clean_phrase)].copy()\n\nprint(\"Muestras limpias:\", len(clean_df))\nprint(\"Porcentaje:\", len(clean_df) / len(train_df) * 100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.439079Z","iopub.execute_input":"2026-03-05T09:15:02.439385Z","iopub.status.idle":"2026-03-05T09:15:02.517107Z","shell.execute_reply.started":"2026-03-05T09:15:02.439359Z","shell.execute_reply":"2026-03-05T09:15:02.515784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"clean_df.iloc[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.518299Z","iopub.execute_input":"2026-03-05T09:15:02.519271Z","iopub.status.idle":"2026-03-05T09:15:02.526405Z","shell.execute_reply.started":"2026-03-05T09:15:02.519204Z","shell.execute_reply":"2026-03-05T09:15:02.52558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# division por participantes, trian, evalu i test no deberían compartir los mismos participantes\nparticipants = clean_df[\"participant_id\"].unique()\nnp.random.shuffle(participants)\n\nn = len(participants)\n\ntrain_ids = participants[:int(0.8 * n)]\nval_ids   = participants[int(0.8 * n):int(0.9 * n)]\ntest_ids  = participants[int(0.9 * n):]\n\ntrain_df_split = clean_df[clean_df[\"participant_id\"].isin(train_ids)]\nval_df_split   = clean_df[clean_df[\"participant_id\"].isin(val_ids)]\ntest_df_split  = clean_df[clean_df[\"participant_id\"].isin(test_ids)]\n\nprint(\"Train:\", len(train_df_split))\nprint(\"Val:  \", len(val_df_split))\nprint(\"Test: \", len(test_df_split))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.527632Z","iopub.execute_input":"2026-03-05T09:15:02.527894Z","iopub.status.idle":"2026-03-05T09:15:02.553092Z","shell.execute_reply.started":"2026-03-05T09:15:02.527869Z","shell.execute_reply":"2026-03-05T09:15:02.552202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"VOCAB_PATH = \"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\"\n\nwith open(VOCAB_PATH) as f:\n    original_letter_to_int = json.load(f)\n\nprint(\"Tamaño vocabulario original:\", len(original_letter_to_int))\nprint(list(original_letter_to_int.items()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.5546Z","iopub.execute_input":"2026-03-05T09:15:02.554862Z","iopub.status.idle":"2026-03-05T09:15:02.571846Z","shell.execute_reply.started":"2026-03-05T09:15:02.554839Z","shell.execute_reply":"2026-03-05T09:15:02.5707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Añadimos el blank para que CTC pueda detectar cuando hay un cambio de letra\nletter_to_int = {}\n\nletter_to_int[\"<blank>\"] = 0\n\n# Desplazamos el resto +1\nfor char, idx in original_letter_to_int.items():\n    letter_to_int[char] = idx + 1\n\nint_to_letter = {v: k for k, v in letter_to_int.items()}\n\nprint(\"Tamaño vocabulario CTC:\", len(letter_to_int))\nprint(list(letter_to_int.items()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.576098Z","iopub.execute_input":"2026-03-05T09:15:02.576523Z","iopub.status.idle":"2026-03-05T09:15:02.582845Z","shell.execute_reply.started":"2026-03-05T09:15:02.576493Z","shell.execute_reply":"2026-03-05T09:15:02.581652Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Nuevo vocabulario filtrado por las letras que vamos a procesar al principio, que no contendran números ni carácteres especiales, por lo que este\n#se reduce a 27 i el Blank, que es lo que el ctc detecta como frames donde no pasa nada, pausas entre letras, transiciones, ruido...\n\nimport string\n\n# letras permitidas\nletters = list(string.ascii_lowercase)\n\n# vocabulario CTC\nletter_to_int = {\"<blank>\": 0}\n\nfor i, ch in enumerate(letters):\n    letter_to_int[ch] = i + 1\n\nint_to_letter = {v: k for k, v in letter_to_int.items()}\n\nprint(\"Tamaño vocabulario:\", len(letter_to_int))\nprint(letter_to_int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.584111Z","iopub.execute_input":"2026-03-05T09:15:02.584605Z","iopub.status.idle":"2026-03-05T09:15:02.606377Z","shell.execute_reply.started":"2026-03-05T09:15:02.584569Z","shell.execute_reply":"2026-03-05T09:15:02.6055Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Encode target","metadata":{}},{"cell_type":"code","source":"def encode_phrase(phrase, letter_to_int):\n    return [letter_to_int[c] for c in phrase if c in letter_to_int]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.60762Z","iopub.execute_input":"2026-03-05T09:15:02.608138Z","iopub.status.idle":"2026-03-05T09:15:02.625467Z","shell.execute_reply.started":"2026-03-05T09:15:02.608095Z","shell.execute_reply":"2026-03-05T09:15:02.624062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Añadir frases codificadas\ntrain_df_split = train_df_split.copy()\nval_df_split   = val_df_split.copy()\n\ntrain_df_split[\"encoded\"] = train_df_split[\"phrase\"].apply(\n    lambda x: encode_phrase(x, letter_to_int)\n)\n\nval_df_split[\"encoded\"] = val_df_split[\"phrase\"].apply(\n    lambda x: encode_phrase(x, letter_to_int)\n)\n\ntrain_df_split[[\"phrase\", \"encoded\"]].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.627009Z","iopub.execute_input":"2026-03-05T09:15:02.627686Z","iopub.status.idle":"2026-03-05T09:15:02.66941Z","shell.execute_reply.started":"2026-03-05T09:15:02.627641Z","shell.execute_reply":"2026-03-05T09:15:02.668682Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Normalize inputs","metadata":{}},{"cell_type":"code","source":"MAX_FRAMES = 160\nRIGHT_HAND_COLS = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.670482Z","iopub.execute_input":"2026-03-05T09:15:02.670835Z","iopub.status.idle":"2026-03-05T09:15:02.675507Z","shell.execute_reply.started":"2026-03-05T09:15:02.670794Z","shell.execute_reply":"2026-03-05T09:15:02.674586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_right_hand_sequence_df(file_id, sequence_id):\n    global RIGHT_HAND_COLS\n\n    path = f\"/kaggle/input/asl-fingerspelling/train_landmarks/{file_id}.parquet\"\n    pq_file = pq.ParquetFile(path)\n\n    # Detectamos columnas una sola vez\n    if RIGHT_HAND_COLS is None:\n        RIGHT_HAND_COLS = [c for c in pq_file.schema.names if \"right_hand\" in c]\n\n    table = pq.read_table(\n        path,\n        filters=[(\"sequence_id\", \"=\", sequence_id)],\n        columns=RIGHT_HAND_COLS\n    )\n\n    X = table.to_pandas()\n    return X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.676814Z","iopub.execute_input":"2026-03-05T09:15:02.677386Z","iopub.status.idle":"2026-03-05T09:15:02.695773Z","shell.execute_reply.started":"2026-03-05T09:15:02.677358Z","shell.execute_reply":"2026-03-05T09:15:02.694739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_right_hand_sequence_array(file_id, sequence_id):\n    return read_right_hand_sequence_df(file_id, sequence_id).values.astype(np.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.697016Z","iopub.execute_input":"2026-03-05T09:15:02.697472Z","iopub.status.idle":"2026-03-05T09:15:02.719028Z","shell.execute_reply.started":"2026-03-05T09:15:02.697435Z","shell.execute_reply":"2026-03-05T09:15:02.717995Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Center hand","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:02.720327Z","iopub.execute_input":"2026-03-05T09:15:02.72071Z","iopub.status.idle":"2026-03-05T09:15:05.154747Z","shell.execute_reply.started":"2026-03-05T09:15:02.720676Z","shell.execute_reply":"2026-03-05T09:15:05.153627Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def center_wrist(pdf):\n    tmp = pdf.copy()\n    for i in range (0,21):\n        tmp[f\"x_right_hand_{i}\"] -= pdf[\"x_right_hand_0\"]\n        tmp[f\"y_right_hand_{i}\"] -= pdf[\"y_right_hand_0\"]\n        tmp[f\"z_right_hand_{i}\"] -= pdf[\"z_right_hand_0\"]\n    return tmp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:05.156094Z","iopub.execute_input":"2026-03-05T09:15:05.156721Z","iopub.status.idle":"2026-03-05T09:15:05.162517Z","shell.execute_reply.started":"2026-03-05T09:15:05.156679Z","shell.execute_reply":"2026-03-05T09:15:05.161539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize_to_box(pdf):\n    x_min = pdf[[f\"x_right_hand_{i}\" for i in range (0,21)]].min().min()\n    x_max = pdf[[f\"x_right_hand_{i}\" for i in range (0,21)]].max().max()\n    y_min = pdf[[f\"y_right_hand_{i}\" for i in range (0,21)]].min().min()\n    y_max = pdf[[f\"y_right_hand_{i}\" for i in range (0,21)]].max().max()\n    \n    ratio = min(1/(x_max-x_min), 1/(y_max-y_min))\n    tmp = pdf.copy()\n    for i in range (0,21):\n        tmp[f\"x_right_hand_{i}\"] -= x_min\n        tmp[f\"x_right_hand_{i}\"] *= ratio\n        tmp[f\"y_right_hand_{i}\"] -= y_min\n        tmp[f\"y_right_hand_{i}\"] *= ratio\n        tmp[f\"z_right_hand_{i}\"] *= ratio\n    return tmp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:05.16381Z","iopub.execute_input":"2026-03-05T09:15:05.164146Z","iopub.status.idle":"2026-03-05T09:15:05.18697Z","shell.execute_reply.started":"2026-03-05T09:15:05.16411Z","shell.execute_reply":"2026-03-05T09:15:05.185812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"row = train_df_split.iloc[20]\nX_pdf = read_right_hand_sequence_df(row[\"file_id\"], row[\"sequence_id\"])\nX_pdf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:05.188588Z","iopub.execute_input":"2026-03-05T09:15:05.188982Z","iopub.status.idle":"2026-03-05T09:15:05.75383Z","shell.execute_reply.started":"2026-03-05T09:15:05.188929Z","shell.execute_reply":"2026-03-05T09:15:05.753035Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Normalize sequence length","metadata":{}},{"cell_type":"code","source":"def normalize_frames(X, max_frames=MAX_FRAMES):\n    T, D = X.shape\n\n    if T > max_frames:\n        return X[:max_frames]\n\n    if T < max_frames:\n        pad = np.zeros((max_frames - T, D), dtype=np.float32)\n        return np.vstack([X, pad])\n\n    return X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:05.75492Z","iopub.execute_input":"2026-03-05T09:15:05.755333Z","iopub.status.idle":"2026-03-05T09:15:05.76154Z","shell.execute_reply.started":"2026-03-05T09:15:05.755305Z","shell.execute_reply":"2026-03-05T09:15:05.760587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#row = train_df_split.iloc[20]\n\nX = read_right_hand_sequence_df(row[\"file_id\"], row[\"sequence_id\"])\n#X = X_centered.values.astype(np.float32)\nXn = normalize_frames(X)\n\nprint(\"Original:\", X.shape)\nprint(\"Normalizado:\", Xn.shape)\nprint(Xn[:2])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:05.762694Z","iopub.execute_input":"2026-03-05T09:15:05.763003Z","iopub.status.idle":"2026-03-05T09:15:05.938175Z","shell.execute_reply.started":"2026-03-05T09:15:05.762966Z","shell.execute_reply":"2026-03-05T09:15:05.937243Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"row = train_df_split.iloc[0]\nfile_id, sequence_id = row[\"file_id\"], row[\"sequence_id\"]\n\npath = f\"/kaggle/input/asl-fingerspelling/train_landmarks/{file_id}.parquet\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:05.939392Z","iopub.execute_input":"2026-03-05T09:15:05.93964Z","iopub.status.idle":"2026-03-05T09:15:05.945281Z","shell.execute_reply.started":"2026-03-05T09:15:05.939617Z","shell.execute_reply":"2026-03-05T09:15:05.9442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pq_file = pq.ParquetFile(path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:05.946736Z","iopub.execute_input":"2026-03-05T09:15:05.947406Z","iopub.status.idle":"2026-03-05T09:15:05.985504Z","shell.execute_reply.started":"2026-03-05T09:15:05.947377Z","shell.execute_reply":"2026-03-05T09:15:05.984204Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def count_valid_frames(X):\n    # Un frame es válido si NO todo es NaN\n    return np.sum(~np.all(np.isnan(X), axis=1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:05.986825Z","iopub.execute_input":"2026-03-05T09:15:05.987162Z","iopub.status.idle":"2026-03-05T09:15:05.992187Z","shell.execute_reply.started":"2026-03-05T09:15:05.987122Z","shell.execute_reply":"2026-03-05T09:15:05.99129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ASLRightHandDataset(Dataset):\n    def __init__(self, df, max_frames=160):\n        self.df = df.reset_index(drop=True)\n        self.max_frames = max_frames\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        X_raw = read_right_hand_sequence_df(row[\"file_id\"], row[\"sequence_id\"]) \n        X_centered = center_wrist(X_raw)\n        \n        input_len = count_valid_frames(X_centered)\n        Y = torch.tensor(row[\"encoded\"], dtype=torch.long)\n        target_len = len(Y)\n\n        if input_len < target_len:\n            return None\n\n        # Pad/truncate frames\n        X = normalize_frames(X_centered, self.max_frames)\n        X = np.nan_to_num(X, nan=0.0)\n        X = torch.tensor(X, dtype=torch.float32)\n        \n        # --- NEW ST-GCN RESHAPING ---\n        # ST-GCNs expect inputs in the shape (Channels, Time, Vertices) \n        # which in this case is (3, 160, 21) representing (x/y/z coordinates, frames, landmarks).\n        \n        # Reshape from (160, 63) to (160, 21, 3) -> (Time, Vertices, Channels)\n        X = X.view(self.max_frames, 21, 3)\n        # Permute to (Channels, Time, Vertices) -> (3, 160, 21)\n        X = X.permute(2, 0, 1) \n        # ----------------------------\n\n        input_len = min(input_len, self.max_frames)\n        return X, Y, input_len, target_len\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:05.993673Z","iopub.execute_input":"2026-03-05T09:15:05.994646Z","iopub.status.idle":"2026-03-05T09:15:06.015984Z","shell.execute_reply.started":"2026-03-05T09:15:05.994601Z","shell.execute_reply":"2026-03-05T09:15:06.014725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_hand_adjacency_matrix():\n    # 21 MediaPipe hand landmarks\n    num_nodes = 21\n    # Physical connections between the joints\n    edges = [\n        (0,1), (1,2), (2,3), (3,4),          # Thumb\n        (0,5), (5,6), (6,7), (7,8),          # Index\n        (5,9), (9,10), (10,11), (11,12),     # Middle\n        (9,13), (13,14), (14,15), (15,16),   # Ring\n        (13,17), (0,17), (17,18), (18,19), (19,20) # Pinky & Palm\n    ]\n    \n    A = np.zeros((num_nodes, num_nodes), dtype=np.float32)\n    for i, j in edges:\n        A[i, j] = 1\n        A[j, i] = 1 # Bidirectional connection\n        \n    # Add self-loops (identity matrix) so nodes look at themselves\n    A = A + np.eye(num_nodes)\n    \n    # Normalize the matrix\n    row_sum = A.sum(axis=1)\n    D_inv = np.diag(row_sum ** -0.5)\n    A_normalized = D_inv @ A @ D_inv\n    \n    return torch.tensor(A_normalized, dtype=torch.float32)\n\nA_matrix = get_hand_adjacency_matrix()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:06.017144Z","iopub.execute_input":"2026-03-05T09:15:06.017506Z","iopub.status.idle":"2026-03-05T09:15:06.073792Z","shell.execute_reply.started":"2026-03-05T09:15:06.01748Z","shell.execute_reply":"2026-03-05T09:15:06.072786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def collate_fn(batch):\n    # Quitamos ejemplos inválidos (None)\n    batch = [b for b in batch if b is not None]\n\n    # Si todo el batch era inválido, devolvemos None\n    if len(batch) == 0:\n        return None\n\n    Xs = []\n    Ys = []\n    in_lens = []\n    tar_lens = []\n\n    for X, Y, in_len, tar_len in batch:\n        Xs.append(X)\n        Ys.append(Y)\n        in_lens.append(in_len)\n        tar_lens.append(tar_len)\n\n    Xs = torch.stack(Xs)\n    Ys = torch.cat(Ys)\n    in_lens = torch.tensor(in_lens, dtype=torch.long)\n    tar_lens = torch.tensor(tar_lens, dtype=torch.long)\n\n    return Xs, Ys, in_lens, tar_lens\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:06.07509Z","iopub.execute_input":"2026-03-05T09:15:06.075445Z","iopub.status.idle":"2026-03-05T09:15:06.082808Z","shell.execute_reply.started":"2026-03-05T09:15:06.07541Z","shell.execute_reply":"2026-03-05T09:15:06.081905Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_df_tiny = train_df_split.sample(200, random_state=0)\n# train_dataset_tiny = ASLRightHandDataset(train_df_tiny)\n\n# print(\"INFO: Creating data loader with TINY size (200)\")\n\n# train_loader_tiny = DataLoader(\n#     train_dataset_tiny,\n#     batch_size=4,\n#     shuffle=True,\n#     collate_fn=collate_fn,\n#     num_workers=0\n# )\n\n# print(\"INFO: Checking sizes\")\n\n# X, Y, in_len, tar_len = next(iter(train_loader_tiny))\n\n# print(\"X:\", X.shape)          # (B, 160, 63)\n# print(\"Y:\", Y.shape)          # (sum of target lengths)\n# print(\"in_len:\", in_len)\n# print(\"tar_len:\", tar_len)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:06.084055Z","iopub.execute_input":"2026-03-05T09:15:06.084477Z","iopub.status.idle":"2026-03-05T09:15:06.104379Z","shell.execute_reply.started":"2026-03-05T09:15:06.084448Z","shell.execute_reply":"2026-03-05T09:15:06.103345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_loader_size = 500\ntrain_df = train_df_split.sample(data_loader_size, random_state=0)\ntrain_dataset = ASLRightHandDataset(train_df)\nprint(f\"INFO: Creating data loader with size: {data_loader_size} \")\n\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=8,\n    shuffle=True,\n    collate_fn=collate_fn,\n    num_workers=0\n)\n\nprint(\"INFO: Checking sizes\")\n\nX, Y, in_len, tar_len = next(iter(train_loader))\n\nprint(\"X:\", X.shape)          # (B, 160, 63)\nprint(\"Y:\", Y.shape)          # (sum of target lengths)\nprint(\"in_len:\", in_len)\nprint(\"tar_len:\", tar_len)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:23:12.640038Z","iopub.execute_input":"2026-03-05T09:23:12.640821Z","iopub.status.idle":"2026-03-05T09:23:13.873263Z","shell.execute_reply.started":"2026-03-05T09:23:12.640784Z","shell.execute_reply":"2026-03-05T09:23:13.872139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class STGCN_Block(nn.Module):\n    def __init__(self, in_channels, out_channels, A, kernel_size=9, dropout=0.3):\n        super().__init__()\n        # Store Adjacency matrix as a non-trainable buffer\n        self.register_buffer('A', A)\n        \n        # Spatial Graph Convolution (implemented as 1x1 Conv over nodes)\n        self.spatial_conv = nn.Conv2d(in_channels, out_channels, kernel_size=1)\n        \n        # Temporal Convolution (Conv2d over the time dimension)\n        padding = ((kernel_size - 1) // 2, 0)\n        self.temporal_conv = nn.Conv2d(out_channels, out_channels, \n                                       kernel_size=(kernel_size, 1), \n                                       padding=padding)\n        \n        self.bn = nn.BatchNorm2d(out_channels)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(dropout)\n\n    def forward(self, x):\n        # x shape: (Batch, Channels, Time, Vertices)\n        \n        # 1. Spatial Graph Convolution\n        x = self.spatial_conv(x)\n        # Multiply node features by the Adjacency Matrix\n        x = torch.einsum('bctv,vw->bctw', x, self.A)\n        \n        # 2. Temporal Convolution\n        x = self.temporal_conv(x)\n        \n        x = self.bn(x)\n        x = self.relu(x)\n        return self.dropout(x)\n\nclass ASL_STGCN(nn.Module):\n    def __init__(self, num_classes, A, in_channels=3, hidden_dim=128):\n        super().__init__()\n        \n        # Three ST-GCN blocks extracting spatial-temporal features\n        self.stgcn1 = STGCN_Block(in_channels, hidden_dim, A)\n        self.stgcn2 = STGCN_Block(hidden_dim, hidden_dim * 2, A)\n        self.stgcn3 = STGCN_Block(hidden_dim * 2, hidden_dim * 2, A)\n        \n        # Final classifier\n        self.classifier = nn.Linear(hidden_dim * 2, num_classes)\n        self.log_softmax = nn.LogSoftmax(dim=-1)\n\n    def forward(self, x):\n        # x shape: (B, 3, 160, 21)\n        x = self.stgcn1(x)\n        x = self.stgcn2(x)\n        x = self.stgcn3(x) # Shape: (B, 256, 160, 21)\n        \n        # Global Average Pooling over the vertices (landmarks) dimension\n        # Averages the 21 landmarks into a single feature vector per time step\n        x = x.mean(dim=3) # Shape: (B, 256, 160)\n        \n        # Permute for linear layer: (B, Time, Channels)\n        x = x.permute(0, 2, 1) # Shape: (B, 160, 256)\n        \n        out = self.classifier(x) # Shape: (B, 160, num_classes)\n        out = self.log_softmax(out)\n        \n        # Permute for CTC Loss: (Time, Batch, num_classes)\n        out = out.permute(1, 0, 2) \n        return out\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:23:15.852175Z","iopub.execute_input":"2026-03-05T09:23:15.853326Z","iopub.status.idle":"2026-03-05T09:23:15.86587Z","shell.execute_reply.started":"2026-03-05T09:23:15.853287Z","shell.execute_reply":"2026-03-05T09:23:15.864785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_classes = len(letter_to_int)\n\n# Instantiate new model\nmodel = ASL_STGCN(\n    num_classes=num_classes, \n    A=A_matrix,\n    in_channels=3,      # x, y, z\n    hidden_dim=128\n).to(DEVICE)\n\nprint(f\"Model: ST-GCN / Parameters: {sum(p.numel() for p in model.parameters()):,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:23:24.02215Z","iopub.execute_input":"2026-03-05T09:23:24.023828Z","iopub.status.idle":"2026-03-05T09:23:24.052547Z","shell.execute_reply.started":"2026-03-05T09:23:24.023773Z","shell.execute_reply":"2026-03-05T09:23:24.051447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.eval()\n\nwith torch.no_grad():\n    outputs = model(X.to(DEVICE))\n\nprint(outputs.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:23:26.078984Z","iopub.execute_input":"2026-03-05T09:23:26.079749Z","iopub.status.idle":"2026-03-05T09:23:26.747854Z","shell.execute_reply.started":"2026-03-05T09:23:26.079715Z","shell.execute_reply":"2026-03-05T09:23:26.747005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"criterion = nn.CTCLoss(\n    blank=letter_to_int[\"<blank>\"],\n    zero_infinity=True,\n    reduction='mean'  # ensure proper reduction\n)\n\nloss = criterion(\n    outputs,    # (T, B, C)\n    Y.to(DEVICE),\n    in_len,\n    tar_len\n)\n\nprint(\"CTC loss:\", loss.item())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:23:28.787247Z","iopub.execute_input":"2026-03-05T09:23:28.788309Z","iopub.status.idle":"2026-03-05T09:23:28.797755Z","shell.execute_reply.started":"2026-03-05T09:23:28.788262Z","shell.execute_reply":"2026-03-05T09:23:28.796995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(model, loader, optimizer, criterion, device):\n    model.train()\n    total_loss = 0.0\n    num_batches = 0\n\n    for batch in loader:\n\n        if batch is None:\n            continue\n\n        X, Y, input_lens, target_lens = batch\n\n        X = X.to(device)\n        Y = Y.to(device)\n        input_lens = input_lens.to(device)\n        target_lens = target_lens.to(device)\n\n        optimizer.zero_grad()\n\n        outputs = model(X)  # (T, B, C)\n\n        loss = criterion(\n            outputs,\n            Y,\n            input_lens,\n            target_lens\n        )\n\n        loss.backward()\n\n\n        optimizer.step()\n\n        total_loss += loss.item()\n        num_batches += 1\n\n    avg_loss = total_loss / num_batches\n\n    return avg_loss\nprint(\"INFO: train_one_epoch method defined\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:23:31.07807Z","iopub.execute_input":"2026-03-05T09:23:31.078893Z","iopub.status.idle":"2026-03-05T09:23:31.089872Z","shell.execute_reply.started":"2026-03-05T09:23:31.078862Z","shell.execute_reply":"2026-03-05T09:23:31.088758Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================================\n# CER IMPLEMENTATION using TorchMetrics\n# =================================\nfrom torchmetrics.text import CharErrorRate\n\ndef ctc_decode(log_probs, int_to_letter, blank_id=0):\n  \"\"\"Simple CTC greedy decoding\"\"\"\n  pred_indices = torch.argmax(log_probs, dim=-1).cpu().numpy()\n  decoded = []\n  previous = None\n\n  for idx in pred_indices:\n      if idx == previous or idx == blank_id:\n          previous = idx\n          continue\n      decoded.append(int_to_letter[idx])\n      previous = idx\n\n  return ''.join(decoded)\n\n\ndef evaluate_cer(model, dataloader, int_to_letter, device, blank_id=0):\n  \"\"\"Evaluate model and return average CER using TorchMetrics\"\"\"\n  model.eval()\n  cer_metric = CharErrorRate()\n\n  all_preds = []\n  all_targets = []\n\n  with torch.no_grad():\n      for batch in dataloader:\n          if batch is None:\n              continue\n\n          X, Y, input_lens, target_lens = batch\n          X = X.to(device)\n\n          # Get predictions\n          outputs = model(X)  # (T, B, C)\n\n          # Decode each sequence\n          batch_size = outputs.shape[1]\n          Y_list = Y.cpu().numpy().tolist()\n          start_idx = 0\n\n          for i in range(batch_size):\n              # Decode prediction\n              pred_text = ctc_decode(outputs[:, i, :], int_to_letter, blank_id)\n              all_preds.append(pred_text)\n\n              # Decode target\n              target_len = target_lens[i]\n              target_indices = Y_list[start_idx:start_idx + target_len]\n              target_text = ''.join([int_to_letter[idx] for idx in target_indices])\n              all_targets.append(target_text)\n              start_idx += target_len\n\n  # Calculate CER using TorchMetrics\n  cer = cer_metric(all_preds, all_targets)\n  return cer.item()\n\nprint(\"INFO: CER implemented\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:15:09.233723Z","iopub.execute_input":"2026-03-05T09:15:09.234399Z","iopub.status.idle":"2026-03-05T09:15:23.016453Z","shell.execute_reply.started":"2026-03-05T09:15:09.23437Z","shell.execute_reply":"2026-03-05T09:15:23.015391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\nstart_time = time.time()\n\n#num_epochs = 10\nnum_epochs = 20\n\ntrain_losses = []\ntrain_cers = []\ngrad_norms = []\nlrate = 1e-3\noptimizer = optim.Adam(model.parameters(), lr=lrate)\n\nprint(\"=\"*60)\nprint(f\"INITIAL TRAINING\")\nprint(\"=\"*60)\n\n# Start a new wandb run to track this training script\nrun = wandb.init(\n    # Team workspace\n    entity=\"inaki-rodriguez-reyes-upc-universidad-peruana-de-ciencia\",\n    # Preview project\n    project=\"asl-fingerspelling-previews\",\n    \n    # Track hyperparameters and run metadata.\n    config={\n        \"name\": \"INITIAL TRAINING (ST-GCN)\",\n        \"notebook\": \"notebook136616d653_v12_gcn\",\n        \"version\": \"v1\",\n        \"learning_rate\": lrate,\n        \"architecture\": \"Spatiotemporal Graph Convolutional Network\",\n        \"dataset\": \"Google-ASL\",\n        \"datasize\": data_loader_size,\n        #\"variante\": \"DROPOUT=0\", #EXPERIMENT_1\n        #\"variante\": \"MAX_FRAMES=300\", #EXPERIMENT_2\n        \"variante\": \"EPOCHS=20+CER\", #EXPERIMENT_3\n        #\"variante\": \"DROPOUT=0.2\", #EXPERIMENT_4\n        \"epochs\": num_epochs,\n    },\n)\n\n\nprint(f\"[Very first training with {data_loader_size} / lr={lrate}]\")\nfor epoch in range(num_epochs):\n    int_time = time.time()\n    loss = train_one_epoch(\n        model,\n        train_loader,\n        optimizer,\n        criterion,\n        DEVICE\n    )\n    train_losses.append(loss)\n    #print(f\"Epoch {epoch+1} -> LOSS = {loss:.2f}\")\n    \n    #CER evaluation\n    cer = evaluate_cer(model, train_loader, int_to_letter, DEVICE)\n    train_cers.append(cer)\n    print(f\"Epoch {epoch+1} -> LOSS = {loss:.2f} - CER = {cer}\")\n    run.log({\"cer\": cer})\n    run.log({\"loss\": loss})\n    end_time = time.time()\n    print(f\"Elapsed time: {end_time - int_time:.2f} secs\")\n\nrun.finish()\n\nfinal_time = time.time()\nprint(\"-\"*60)\nprint(f\"TOTAL time: {(final_time - start_time)/60:.2f} mins\")\nprint(\"-\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:23:35.862722Z","iopub.execute_input":"2026-03-05T09:23:35.86343Z","execution_failed":"2026-03-05T09:27:58.077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(6,4))\nplt.plot(train_losses, marker=\"o\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.title(f\"Training Loss over {num_epochs} Epochs / lr={lrate}\")\nplt.grid(True)\nplt.show()\n\nprint(\"-\"*60)\nprint(f\"LOSS TREND\")\nprint(f\"First loss: {train_losses[0]:.2f}\")\nprint(f\"Last loss: {train_losses[-1]:.2f}\")\nprint(f\"Loss decreased: {train_losses[0] - train_losses[-1]:.2f}\")\nprint(\"-\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:19:23.602035Z","iopub.status.idle":"2026-03-05T09:19:23.602488Z","shell.execute_reply.started":"2026-03-05T09:19:23.602331Z","shell.execute_reply":"2026-03-05T09:19:23.602352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"INFO: [END_OF_NOTEBOOK]\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-05T09:19:23.604285Z","iopub.status.idle":"2026-03-05T09:19:23.604912Z","shell.execute_reply.started":"2026-03-05T09:19:23.604739Z","shell.execute_reply":"2026-03-05T09:19:23.60476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}