{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":52950,"databundleVersionId":5973250,"sourceType":"competition"},{"sourceId":11694503,"sourceType":"datasetVersion","datasetId":7340011}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\nimport pyarrow.parquet as pq\nimport tensorflow as tf\nimport json\nimport matplotlib\n# Đảm bảo matplotlib sử dụng backend không tương tác nếu chạy trong môi trường không có GUI\n# matplotlib.use('Agg') # Bỏ comment nếu cần\nimport matplotlib.pyplot as plt\nimport random\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import TerminateOnNaN, EarlyStopping, ReduceLROnPlateau\nimport numpy as np\nimport pandas as pd\n\n# --- 0. GLOBAL CONSTANTS AND CONFIGURATION ---\n\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE]\nFEATURE_COLUMNS = X + Y + Z\n\nRHAND_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"right_hand\" in col]\nLHAND_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"left_hand\" in col]\nRPOSE_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"pose\" in col and int(col.split('_')[-1]) in RPOSE]\nLPOSE_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"pose\" in col and int(col.split('_')[-1]) in LPOSE]\n\nFRAME_LEN = 128\nTARGET_MAXLEN = 64 # Độ dài tối đa của chuỗi nhãn sau khi padding\n\nCHAR_TO_NUM_PATH = \"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\"\nif not os.path.exists(CHAR_TO_NUM_PATH):\n    print(f\"Warning: Character map file not found at {CHAR_TO_NUM_PATH}. Using a dummy map.\")\n    char_to_num_orig = {chr(ord('a') + i): i for i in range(26)}\n    char_to_num_orig[' '] = 26\nelse:\n    with open (CHAR_TO_NUM_PATH, \"r\") as f:\n        char_to_num_orig = json.load(f)\n\n# --- Vocabulary cho CTC ---\n_char_to_num_intermediate = {}\ncurrent_max_original_idx = -1\nfor char, idx_val in char_to_num_orig.items():\n    try:\n        idx = int(idx_val)\n        _char_to_num_intermediate[char] = idx\n        current_max_original_idx = max(current_max_original_idx, idx)\n    except ValueError:\n        print(f\"Warning: Could not convert index '{idx_val}' for char '{char}' to int. Skipping.\")\n\npad_token = 'P'\nunknown_token = '?'\n\nunique_chars_from_orig = list(_char_to_num_intermediate.keys())\nif unknown_token not in unique_chars_from_orig:\n    unique_chars_from_orig.append(unknown_token)\nsorted_predictable_chars = sorted(list(set(unique_chars_from_orig)))\n\nctc_char_to_num = {}\nfor i, char in enumerate(sorted_predictable_chars):\n    ctc_char_to_num[char] = i\n\nctc_num_to_char = {v: k for k, v in ctc_char_to_num.items()}\n\nNUM_PREDICTABLE_CHARS = len(ctc_char_to_num)\nBLANK_INDEX = NUM_PREDICTABLE_CHARS\nCTC_NUM_CLASSES = NUM_PREDICTABLE_CHARS + 1\npad_token_idx_for_padding = -1\n\nprint(f\"--- Vocabulary Debug for CTC ---\")\nprint(f\"ctc_char_to_num (ký tự dự đoán được map tới 0..N-1): {ctc_char_to_num}\")\nprint(f\"ctc_num_to_char: {ctc_num_to_char}\")\nprint(f\"unknown_token is mapped to ctc_char_to_num index: {ctc_char_to_num.get(unknown_token, 'NOT IN VOCAB (ERROR)')}\")\nprint(f\"NUM_PREDICTABLE_CHARS (N): {NUM_PREDICTABLE_CHARS}\")\nprint(f\"BLANK_INDEX (sẽ là N): {BLANK_INDEX}\")\nprint(f\"CTC_NUM_CLASSES (N+1, cho lớp Dense cuối, bao gồm blank): {CTC_NUM_CLASSES}\")\nprint(f\"pad_token_idx_for_padding (dùng để pad nhãn): {pad_token_idx_for_padding}\")\nprint(f\"--- End Vocabulary Debug ---\")\n\ndefault_lookup_value = ctc_char_to_num.get(unknown_token)\nif default_lookup_value is None:\n    print(f\"CRITICAL WARNING: unknown_token '{unknown_token}' not found in ctc_char_to_num. Defaulting lookup to 0.\")\n    default_lookup_value = 0 if NUM_PREDICTABLE_CHARS > 0 else -1\n\nctc_table = tf.lookup.StaticHashTable(\n    initializer=tf.lookup.KeyValueTensorInitializer(\n        keys=list(ctc_char_to_num.keys()),\n        values=tf.constant(list(ctc_char_to_num.values()), dtype=tf.int32),\n    ),\n    default_value=tf.constant(default_lookup_value, dtype=tf.int32),\n    name=\"ctc_char_to_num_lookup\"\n)\n\n# --- 1. DATA PREPROCESSING FUNCTIONS ---\ndef pre_process(x):\n    x_rh = tf.gather(x, indices=RHAND_IDX[:21], axis=1)\n    y_rh = tf.gather(x, indices=RHAND_IDX[21:42], axis=1)\n    z_rh = tf.gather(x, indices=RHAND_IDX[42:63], axis=1)\n    rhand = tf.stack([x_rh, y_rh, z_rh], axis=-1)\n    x_lh = tf.gather(x, indices=LHAND_IDX[:21], axis=1)\n    y_lh = tf.gather(x, indices=LHAND_IDX[21:42], axis=1)\n    z_lh = tf.gather(x, indices=LHAND_IDX[42:63], axis=1)\n    lhand = tf.stack([x_lh, y_lh, z_lh], axis=-1)\n    x_rp = tf.gather(x, indices=RPOSE_IDX[:5], axis=1)\n    y_rp = tf.gather(x, indices=RPOSE_IDX[5:10], axis=1)\n    z_rp = tf.gather(x, indices=RPOSE_IDX[10:15], axis=1)\n    rpose_data = tf.stack([x_rp, y_rp, z_rp], axis=-1)\n    x_lp = tf.gather(x, indices=LPOSE_IDX[:5], axis=1)\n    y_lp = tf.gather(x, indices=LPOSE_IDX[5:10], axis=1)\n    z_lp = tf.gather(x, indices=LPOSE_IDX[10:15], axis=1)\n    lpose_data = tf.stack([x_lp, y_lp, z_lp], axis=-1)\n\n    total_rhand_nans = tf.reduce_sum(tf.cast(tf.math.is_nan(rhand), tf.float32))\n    total_lhand_nans = tf.reduce_sum(tf.cast(tf.math.is_nan(lhand), tf.float32))\n    is_left_dominant = total_rhand_nans > total_lhand_nans\n\n    dominant_hand_data_raw = tf.cond(is_left_dominant, lambda: tf.concat([1.0 - lhand[..., :1], lhand[..., 1:]], axis=-1), lambda: rhand)\n    dominant_pose_data_raw = tf.cond(is_left_dominant, lambda: tf.concat([1.0 - lpose_data[..., :1], lpose_data[..., 1:]], axis=-1), lambda: rpose_data)\n\n    dominant_hand_data_filled = tf.where(tf.math.is_nan(dominant_hand_data_raw), tf.zeros_like(dominant_hand_data_raw), dominant_hand_data_raw)\n    dominant_pose_data_filled = tf.where(tf.math.is_nan(dominant_pose_data_raw), tf.zeros_like(dominant_pose_data_raw), dominant_pose_data_raw)\n\n    mean_hand = tf.math.reduce_mean(dominant_hand_data_filled, axis=1, keepdims=True)\n    std_hand = tf.math.reduce_std(dominant_hand_data_filled, axis=1, keepdims=True)\n    hand_normalized = (dominant_hand_data_filled - mean_hand) / (std_hand + 1e-6)\n    mean_pose = tf.math.reduce_mean(dominant_pose_data_filled, axis=1, keepdims=True)\n    std_pose = tf.math.reduce_std(dominant_pose_data_filled, axis=1, keepdims=True)\n    pose_normalized = (dominant_pose_data_filled - mean_pose) / (std_pose + 1e-6)\n\n    processed_hand = tf.where(tf.math.is_nan(hand_normalized), tf.zeros_like(hand_normalized), hand_normalized)\n    processed_pose = tf.where(tf.math.is_nan(pose_normalized), tf.zeros_like(pose_normalized), pose_normalized)\n    combined_features = tf.concat([processed_hand, processed_pose], axis=1)\n\n    current_frames = tf.shape(combined_features)[0]\n    target_len = FRAME_LEN\n    expected_landmarks = 21 + 5\n    expected_coords = 3\n    \n    reshaped_for_resize = tf.reshape(combined_features, [1, current_frames, expected_landmarks * expected_coords])\n    resized_features = tf.image.resize(reshaped_for_resize, [1, target_len], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n    final_features_flat = tf.reshape(resized_features, [target_len, expected_landmarks * expected_coords])\n    final_features_flat.set_shape([target_len, expected_landmarks * expected_coords])\n    return final_features_flat\n\ndef decode_fn(record_bytes):\n    schema = {COL: tf.io.VarLenFeature(dtype=tf.float32) for COL in FEATURE_COLUMNS}\n    schema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n    features = tf.io.parse_single_example(record_bytes, schema)\n    phrase = features[\"phrase\"]\n    landmarks_list = [tf.sparse.to_dense(features[col]) for col in FEATURE_COLUMNS]\n    landmarks = tf.transpose(tf.stack(landmarks_list))\n    return landmarks, phrase\n\ndef convert_fn_ctc(landmarks, phrase_str):\n    processed_landmarks = pre_process(landmarks)\n    phrase_chars = tf.strings.bytes_split(phrase_str)\n    phrase_ids_ctc = ctc_table.lookup(phrase_chars)\n    label_length = tf.shape(phrase_ids_ctc)[0]\n    padding_size = TARGET_MAXLEN - label_length\n    padding_size = tf.maximum(0, padding_size)\n    phrase_padded_ctc = tf.pad(phrase_ids_ctc, paddings=[[0, padding_size]], mode='CONSTANT', constant_values=tf.cast(pad_token_idx_for_padding, dtype=tf.int32))\n    phrase_padded_ctc = phrase_padded_ctc[:TARGET_MAXLEN]\n    phrase_padded_ctc.set_shape([TARGET_MAXLEN])\n    input_length_val = FRAME_LEN // 4\n    return processed_landmarks, phrase_padded_ctc, tf.cast(label_length, dtype=tf.int32), tf.cast(input_length_val, dtype=tf.int32)\n\n# --- 2. MODEL ARCHITECTURE CLASSES ---\nclass LandmarkEmbedding(layers.Layer):\n    def __init__(self, num_hid, dropout_rate=0.2):\n        super().__init__()\n        self.input_projection = layers.Dense(num_hid, activation=None, use_bias=False, name=\"landmark_input_proj\")\n        self.layer_norm_proj = layers.LayerNormalization(name=\"landmark_proj_layernorm\")\n        self.proj_activation = layers.Activation('relu', name=\"landmark_proj_relu\")\n        self.conv1 = tf.keras.layers.Conv1D(num_hid, kernel_size=5, strides=2, padding=\"same\", activation=None, use_bias=False, name=\"landmark_conv1\")\n        self.norm1 = layers.LayerNormalization(name=\"landmark_conv1_layernorm\")\n        self.act1 = layers.Activation('relu', name=\"landmark_conv1_relu\")\n        self.drop1 = layers.Dropout(dropout_rate, name=\"landmark_conv1_dropout\")\n        self.conv2 = tf.keras.layers.Conv1D(num_hid, kernel_size=5, strides=2, padding=\"same\", activation=None, use_bias=False, name=\"landmark_conv2\")\n        self.norm2 = layers.LayerNormalization(name=\"landmark_conv2_layernorm\")\n        self.act2 = layers.Activation('relu', name=\"landmark_conv2_relu\")\n        self.drop2 = layers.Dropout(dropout_rate, name=\"landmark_conv2_dropout\")\n        self.conv3 = tf.keras.layers.Conv1D(num_hid, kernel_size=3, strides=1, padding=\"same\", activation=None, use_bias=False, name=\"landmark_conv3_stride1\")\n        self.norm3 = layers.LayerNormalization(name=\"landmark_conv3_layernorm\")\n        self.act3 = layers.Activation('relu', name=\"landmark_conv3_relu\")\n        self.drop3 = layers.Dropout(dropout_rate, name=\"landmark_conv3_dropout\")\n    def call(self, x, training=False):\n        x = self.input_projection(x); x = self.layer_norm_proj(x, training=training); x = self.proj_activation(x)\n        x = self.conv1(x); x = self.norm1(x, training=training); x = self.act1(x); x = self.drop1(x, training=training)\n        x = self.conv2(x); x = self.norm2(x, training=training); x = self.act2(x); x = self.drop2(x, training=training)\n        x = self.conv3(x); x = self.norm3(x, training=training); x = self.act3(x); x = self.drop3(x, training=training)\n        return x\n\nclass CTCPredictionModel(keras.Model):\n    def __init__(self, num_hid, ctc_num_classes, dropout_rate=0.2, **kwargs):\n        super().__init__(**kwargs)\n        self.num_hid = num_hid\n        self.ctc_num_classes = ctc_num_classes\n        self.loss_metric = keras.metrics.Mean(name=\"loss\")\n        self.edit_dist_metric = keras.metrics.Mean(name=\"edit_dist\")\n        self.landmark_embedding = LandmarkEmbedding(num_hid=num_hid, dropout_rate=dropout_rate)\n        self.encoder_bilstm1 = layers.Bidirectional(layers.LSTM(num_hid, return_sequences=True, dropout=dropout_rate), name=\"encoder_bilstm_1\")\n        self.encoder_bilstm2 = layers.Bidirectional(layers.LSTM(num_hid, return_sequences=True, dropout=dropout_rate), name=\"encoder_bilstm_2\")\n        self.ctc_output_dense = layers.Dense(self.ctc_num_classes, name=\"ctc_classifier_dense\")\n    def call(self, source_input, training=False):\n        embedded_source = self.landmark_embedding(source_input, training=training)\n        encoder_out = self.encoder_bilstm1(embedded_source, training=training)\n        encoder_out = self.encoder_bilstm2(encoder_out, training=training)\n        ctc_logits = self.ctc_output_dense(encoder_out)\n        return ctc_logits\n\n    def train_step(self, batch):\n        source, y_true, label_length, input_length = batch\n        y_true = tf.cast(y_true, dtype=tf.int32)\n        input_length_for_loss = tf.cast(tf.reshape(input_length, [-1, 1]), dtype=tf.int32)\n        label_length_for_loss = tf.cast(tf.reshape(label_length, [-1, 1]), dtype=tf.int32)\n\n        with tf.GradientTape() as tape:\n            y_pred_logits = self(source, training=True) \n            y_pred_logits_time_major = tf.transpose(y_pred_logits, perm=[1, 0, 2])\n            y_pred_softmax_batch_major = tf.nn.softmax(y_pred_logits, axis=-1)\n\n            raw_loss = tf.keras.backend.ctc_batch_cost(\n                y_true, y_pred_softmax_batch_major, input_length_for_loss, label_length_for_loss\n            )\n            finite_loss_values = tf.boolean_mask(raw_loss, tf.math.is_finite(raw_loss))\n            \n            loss = tf.cond(\n                tf.size(finite_loss_values) > 0,\n                lambda: tf.reduce_mean(finite_loss_values),\n                lambda: tf.constant(0.0) \n            )\n\n        grads = tape.gradient(loss, self.trainable_variables)\n        if not any(g is None for g in grads):\n            self.optimizer.apply_gradients(zip(grads, self.trainable_variables))\n\n        self.loss_metric.update_state(loss)\n\n        input_length_for_decode = tf.reshape(input_length, [-1])\n        \n        decoded_st_tuple_list, _ = tf.nn.ctc_greedy_decoder(\n            inputs=y_pred_logits_time_major,\n            sequence_length=input_length_for_decode,\n            merge_repeated=True\n        )\n        decoded_predictions_st_tuple = decoded_st_tuple_list[0]\n        decoded_predictions_sparse = tf.SparseTensor(\n            indices=decoded_predictions_st_tuple.indices,\n            values=decoded_predictions_st_tuple.values,\n            dense_shape=decoded_predictions_st_tuple.dense_shape\n        )\n        \n        y_true_for_sparse = tf.cast(y_true, dtype=tf.int64)\n        label_length_for_sparse = tf.reshape(label_length, [-1])\n        indices = tf.where(tf.sequence_mask(label_length_for_sparse, maxlen=tf.shape(y_true_for_sparse)[1]))\n        values = tf.gather_nd(y_true_for_sparse, indices)\n        \n        max_len_in_batch = tf.cond(tf.reduce_sum(label_length_for_sparse) > 0,\n                                   lambda: tf.reduce_max(label_length_for_sparse),\n                                   lambda: tf.constant(0, dtype=tf.int32))\n\n        is_empty_batch_labels = tf.reduce_sum(label_length_for_sparse) == 0\n        \n        def get_dense_shape_with_labels():\n            return tf.stack([tf.cast(tf.shape(y_true_for_sparse)[0], dtype=tf.int64), \n                             tf.cast(max_len_in_batch, dtype=tf.int64)])\n        def get_dense_shape_empty_labels():\n            return tf.stack([tf.cast(tf.shape(y_true_for_sparse)[0], dtype=tf.int64), \n                             tf.constant(0, dtype=tf.int64)])\n\n        dense_shape = tf.cond(is_empty_batch_labels, get_dense_shape_empty_labels, get_dense_shape_with_labels)\n        y_true_sparse = tf.SparseTensor(indices, values, dense_shape)\n\n        edit_dist = tf.edit_distance(decoded_predictions_sparse, y_true_sparse, normalize=False)\n        \n        finite_edit_dist_indices = tf.where(tf.math.is_finite(edit_dist))\n        valid_edit_dist_values = tf.gather_nd(edit_dist, finite_edit_dist_indices)\n        \n        mean_edit_dist = tf.cond(\n            tf.size(valid_edit_dist_values) > 0,\n            lambda: tf.reduce_mean(valid_edit_dist_values),\n            lambda: tf.constant(0.0, dtype=tf.float32) # Đảm bảo dtype\n        )\n        \n        should_update_metric = tf.size(valid_edit_dist_values) > 0\n        def update_op():\n            self.edit_dist_metric.update_state(mean_edit_dist)\n            return tf.constant(True) \n        def no_update_op():\n            return tf.constant(False) \n        _ = tf.cond(should_update_metric, update_op, no_update_op)\n\n\n        return {\"loss\": self.loss_metric.result(), \"edit_dist\": self.edit_dist_metric.result()}\n\n    def test_step(self, batch):\n        source, y_true, label_length, input_length = batch\n        y_true = tf.cast(y_true, dtype=tf.int32)\n        input_length_for_loss = tf.cast(tf.reshape(input_length, [-1, 1]), dtype=tf.int32)\n        label_length_for_loss = tf.cast(tf.reshape(label_length, [-1, 1]), dtype=tf.int32)\n\n        y_pred_logits = self(source, training=False)\n        y_pred_logits_time_major = tf.transpose(y_pred_logits, perm=[1, 0, 2])\n        y_pred_softmax_batch_major = tf.nn.softmax(y_pred_logits, axis=-1)\n        \n        raw_loss = tf.keras.backend.ctc_batch_cost(\n            y_true, y_pred_softmax_batch_major, input_length_for_loss, label_length_for_loss\n        )\n        finite_loss_values = tf.boolean_mask(raw_loss, tf.math.is_finite(raw_loss))\n        \n        loss = tf.cond(\n            tf.size(finite_loss_values) > 0,\n            lambda: tf.reduce_mean(finite_loss_values),\n            lambda: tf.constant(0.0)\n        )\n        self.loss_metric.update_state(loss)\n\n        input_length_for_decode = tf.reshape(input_length, [-1])\n        decoded_st_tuple_list, _ = tf.nn.ctc_greedy_decoder(\n            inputs=y_pred_logits_time_major,\n            sequence_length=input_length_for_decode,\n            merge_repeated=True\n        )\n        decoded_predictions_st_tuple = decoded_st_tuple_list[0]\n        decoded_predictions_sparse = tf.SparseTensor(\n            indices=decoded_predictions_st_tuple.indices,\n            values=decoded_predictions_st_tuple.values,\n            dense_shape=decoded_predictions_st_tuple.dense_shape\n        )\n\n        y_true_for_sparse = tf.cast(y_true, dtype=tf.int64)\n        label_length_for_sparse = tf.reshape(label_length, [-1])\n        indices = tf.where(tf.sequence_mask(label_length_for_sparse, maxlen=tf.shape(y_true_for_sparse)[1]))\n        values = tf.gather_nd(y_true_for_sparse, indices)\n\n        max_len_in_batch = tf.cond(tf.reduce_sum(label_length_for_sparse) > 0,\n                                   lambda: tf.reduce_max(label_length_for_sparse),\n                                   lambda: tf.constant(0, dtype=tf.int32))\n                                   \n        is_empty_batch_labels = tf.reduce_sum(label_length_for_sparse) == 0\n        \n        def get_dense_shape_with_labels():\n            return tf.stack([tf.cast(tf.shape(y_true_for_sparse)[0], dtype=tf.int64), \n                             tf.cast(max_len_in_batch, dtype=tf.int64)])\n        def get_dense_shape_empty_labels():\n            return tf.stack([tf.cast(tf.shape(y_true_for_sparse)[0], dtype=tf.int64), \n                             tf.constant(0, dtype=tf.int64)])\n        \n        dense_shape = tf.cond(is_empty_batch_labels, get_dense_shape_empty_labels, get_dense_shape_with_labels)\n        y_true_sparse = tf.SparseTensor(indices, values, dense_shape)\n        \n        edit_dist = tf.edit_distance(decoded_predictions_sparse, y_true_sparse, normalize=False)\n        finite_edit_dist_indices = tf.where(tf.math.is_finite(edit_dist))\n        valid_edit_dist_values = tf.gather_nd(edit_dist, finite_edit_dist_indices)\n\n        mean_edit_dist = tf.cond(\n            tf.size(valid_edit_dist_values) > 0,\n            lambda: tf.reduce_mean(valid_edit_dist_values),\n            lambda: tf.constant(0.0, dtype=tf.float32) # Đảm bảo dtype\n        )\n        \n        should_update_metric = tf.size(valid_edit_dist_values) > 0\n        def update_op_test():\n            self.edit_dist_metric.update_state(mean_edit_dist)\n            return tf.constant(True) \n        def no_update_op_test():\n            return tf.constant(False) \n        _ = tf.cond(should_update_metric, update_op_test, no_update_op_test)\n                \n        return {\"loss\": self.loss_metric.result(), \"edit_dist\": self.edit_dist_metric.result()}\n\n    def ctc_generate_greedy(self, source_input, ctc_num_to_char_map_local, blank_idx_local):\n        batch_size_val = tf.shape(source_input)[0]\n        static_batch_size = tf.get_static_value(batch_size_val)\n        if static_batch_size is not None and static_batch_size == 0:\n             return [\"\"] * 0\n        \n        y_pred_logits = self(source_input, training=False)\n        y_pred_logits_time_major = tf.transpose(y_pred_logits, perm=[1, 0, 2])\n\n        input_len_val = FRAME_LEN // 4\n        input_lengths_for_decode = tf.ones(shape=(batch_size_val,), dtype=tf.int32) * input_len_val\n        \n        decoded_st_tuple_list, _ = tf.nn.ctc_greedy_decoder(\n            inputs=y_pred_logits_time_major,\n            sequence_length=input_lengths_for_decode,\n            merge_repeated=True\n        )\n        decoded_sparse = tf.SparseTensor(\n            indices=decoded_st_tuple_list[0].indices,\n            values=decoded_st_tuple_list[0].values,\n            dense_shape=decoded_st_tuple_list[0].dense_shape\n        )\n        \n        non_char_default_value = tf.cast(blank_idx_local + 100, dtype=tf.int64)\n        \n        try:\n            decoded_dense_np = tf.sparse.to_dense(decoded_sparse, default_value=non_char_default_value).numpy()\n        except AttributeError:\n            if static_batch_size is not None:\n                return [\"<graph_mode_decode_issue_numpy>\"] * static_batch_size\n            else:\n                return [\"<graph_mode_decode_issue_numpy_unknown_batch>\"]\n\n        output_texts = []\n        for i in range(decoded_dense_np.shape[0]):\n            default_val_for_compare = non_char_default_value.numpy() if hasattr(non_char_default_value, 'numpy') else non_char_default_value\n            sequence_indices = [idx for idx in decoded_dense_np[i] if idx != blank_idx_local and idx != default_val_for_compare]\n            sequence_chars = [ctc_num_to_char_map_local.get(idx, '?') for idx in sequence_indices]\n            output_texts.append(\"\".join(sequence_chars))\n        return output_texts\n    @property\n    def metrics(self): return [self.loss_metric, self.edit_dist_metric]\n\n# --- 3. CALLBACKS ---\nclass DisplayOutputsCTC(keras.callbacks.Callback):\n    def __init__(self, batch_data, ctc_n2c_map, blank_idx):\n        super().__init__()\n        if batch_data is not None:\n            self.source_data = batch_data[0]; self.target_labels_padded = batch_data[1].numpy(); self.target_label_lengths = batch_data[2].numpy()\n        else: self.source_data = None; self.target_labels_padded = None; self.target_label_lengths = None\n        self.ctc_num_to_char_map_cb = ctc_n2c_map; self.blank_idx_cb = blank_idx\n    def on_epoch_end(self, epoch, logs=None):\n        if epoch % 5 != 0: return\n        if self.source_data is None or self.target_labels_padded is None or self.target_label_lengths is None:\n            print(\"DisplayOutputsCTC: No batch data for display.\"); return\n        \n        source_shape = tf.shape(self.source_data)\n        batch_size_disp_tensor = source_shape[0]\n        batch_size_disp_val = tf.get_static_value(batch_size_disp_tensor)\n        if batch_size_disp_val is None:\n            try:\n                batch_size_disp_val = batch_size_disp_tensor.numpy()\n            except AttributeError:\n                print(\"DisplayOutputsCTC: Could not determine batch size for display.\")\n                return\n\n        if batch_size_disp_val == 0 : return\n        \n        predictions_text = self.model.ctc_generate_greedy(self.source_data, self.ctc_num_to_char_map_cb, self.blank_idx_cb)\n        print(f\"\\n--- Epoch {epoch+1} Sample Predictions (CTC) ---\")\n        for i in range(min(batch_size_disp_val, 3)):\n            target_text = \"\"\n            if self.target_label_lengths[i] > 0:\n                true_label_indices = self.target_labels_padded[i, :self.target_label_lengths[i]]\n                target_text_list = [self.ctc_num_to_char_map_cb.get(val, '?') for val in true_label_indices if val != pad_token_idx_for_padding and val < self.blank_idx_cb]\n                target_text = \"\".join(target_text_list)\n            \n            if i < len(predictions_text):\n                prediction_text = predictions_text[i]\n                print(f\"Target    : {target_text}\"); print(f\"Prediction: {prediction_text}\\n\")\n            else:\n                print(f\"Target    : {target_text}\"); print(f\"Prediction: <error_generating_prediction_for_sample_{i}>\\n\")\n\n        print(f\"--- End of Sample Predictions (CTC) ---\\n\")\n\n# --- 4. DATASET LOADING AND PREPARATION ---\nTRAIN_CSV_PATH = '/kaggle/input/asl-fingerspelling/train.csv'\nTFRECORDS_BASE_PATH = '/kaggle/input/pre-data-fsp/new_data/'\nif not os.path.exists(TRAIN_CSV_PATH): exit(\"ERROR: Train CSV not found\")\nif not os.path.exists(TFRECORDS_BASE_PATH) or (os.path.isdir(TFRECORDS_BASE_PATH) and not os.listdir(TFRECORDS_BASE_PATH)): exit(\"ERROR: TFRecords directory not found or empty\")\ndataset_df = pd.read_csv(TRAIN_CSV_PATH); tf_records_paths = dataset_df.file_id.map(lambda x: os.path.join(TFRECORDS_BASE_PATH, f'{x}.tfrecord')).unique()\ntf_records_paths = [path for path in tf_records_paths if os.path.exists(path)]\nif not tf_records_paths: exit(\"No TFRecord files found.\")\nprint(f\"Found {len(tf_records_paths)} TFRecord files.\"); random.shuffle(tf_records_paths)\nBATCH_SIZE = 64; TRAIN_SPLIT_RATIO = 0.9; EPOCHS = 100; LEARNING_RATE = 1e-3; NUM_HIDDEN_UNITS = 384; DROPOUT_RATE = 0.3\ntrain_len_count = int(TRAIN_SPLIT_RATIO * len(tf_records_paths))\nif len(tf_records_paths) > 1:\n    if train_len_count == len(tf_records_paths) and train_len_count > 0: train_len_count -=1\n    if train_len_count == 0 and len(tf_records_paths) > 0 : train_len_count = 1\nelif len(tf_records_paths) == 1: train_len_count = 1\nelse: train_len_count = 0\nprint(f\"Using {train_len_count} files for training, {len(tf_records_paths) - train_len_count} for validation.\")\npadded_shapes_ctc = ([FRAME_LEN, 26 * 3], [TARGET_MAXLEN], [], [])\ntrain_ds = None\nif train_len_count > 0:\n    train_ds = tf.data.TFRecordDataset(tf_records_paths[:train_len_count]).shuffle(buffer_size=min(train_len_count * 10, 2048)).map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE).map(convert_fn_ctc, num_parallel_calls=tf.data.AUTOTUNE).padded_batch(BATCH_SIZE, padded_shapes=padded_shapes_ctc, drop_remainder=True).prefetch(buffer_size=tf.data.AUTOTUNE).cache()\nvalid_ds = None; num_val_files = len(tf_records_paths) - train_len_count\nif num_val_files > 0:\n    valid_ds = tf.data.TFRecordDataset(tf_records_paths[train_len_count:]).map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE).map(convert_fn_ctc, num_parallel_calls=tf.data.AUTOTUNE).padded_batch(BATCH_SIZE, padded_shapes=padded_shapes_ctc, drop_remainder=True).prefetch(buffer_size=tf.data.AUTOTUNE).cache()\n\n# --- 5. MODEL TRAINING ---\ncallbacks_list_train = [TerminateOnNaN()]; batch_for_display_cb_ctc = None\nif valid_ds:\n    try: batch_for_display_cb_ctc = next(iter(valid_ds.take(1)))\n    except (StopIteration, tf.errors.OutOfRangeError): print(\"Warning: Validation dataset empty or could not get sample for DisplayOutputsCTC.\"); pass\nif batch_for_display_cb_ctc is None and train_ds:\n    try: batch_for_display_cb_ctc = next(iter(train_ds.take(1)))\n    except (StopIteration, tf.errors.OutOfRangeError): print(\"Warning: Training dataset also empty or could not get sample for DisplayOutputsCTC.\"); pass\nif batch_for_display_cb_ctc is not None:\n    display_cb_instance_ctc = DisplayOutputsCTC(batch_for_display_cb_ctc, ctc_num_to_char, BLANK_INDEX); callbacks_list_train.append(display_cb_instance_ctc)\nelse: print(\"DisplayOutputsCTC callback disabled as no suitable batch could be obtained.\")\nmonitor_metric = 'val_edit_dist' if valid_ds else 'edit_dist'\ncallbacks_list_train.append(EarlyStopping(monitor=monitor_metric, patience=20, restore_best_weights=True, verbose=1, mode='min'))\ncallbacks_list_train.append(ReduceLROnPlateau(monitor=monitor_metric, factor=0.5, patience=7, min_lr=1e-6, verbose=1, mode='min'))\nmodel_instance_ctc = CTCPredictionModel(num_hid=NUM_HIDDEN_UNITS, ctc_num_classes=CTC_NUM_CLASSES, dropout_rate=DROPOUT_RATE)\noptimizer_instance = keras.optimizers.Adam(learning_rate=LEARNING_RATE); model_instance_ctc.compile(optimizer=optimizer_instance)\nsample_for_build_ctc = None\nif train_ds:\n    try: sample_batch_train = next(iter(train_ds.take(1))); sample_for_build_ctc = sample_batch_train[0]\n    except (StopIteration, tf.errors.OutOfRangeError): print(\"Warning: Training dataset empty for model build.\"); pass\n    except Exception as e: print(f\"Warning: Error taking sample from training dataset for model build: {e}\"); pass\nif sample_for_build_ctc is None and valid_ds:\n    try: sample_batch_valid = next(iter(valid_ds.take(1))); sample_for_build_ctc = sample_batch_valid[0]\n    except (StopIteration, tf.errors.OutOfRangeError): print(\"Warning: Validation dataset empty for model build.\"); pass\n    except Exception as e: print(f\"Warning: Error taking sample from validation dataset for model build: {e}\"); pass\nif sample_for_build_ctc is not None:\n    try: model_instance_ctc(sample_for_build_ctc); model_instance_ctc.summary()\n    except Exception as e: print(f\"Error during model build or summary: {e}\")\nelse: print(\"Could not obtain a sample to build the CTC model and print summary.\")\nprint(f\"CTC Model compiled. Training: Batch={BATCH_SIZE}, LR={LEARNING_RATE}, Hidden={NUM_HIDDEN_UNITS}, CTC Classes={CTC_NUM_CLASSES}\")\nif train_ds:\n    history_obj_ctc = model_instance_ctc.fit(train_ds, validation_data=valid_ds, epochs=EPOCHS, callbacks=callbacks_list_train, verbose=1)\n    print(\"\\nCTC Training finished.\")\n    if history_obj_ctc and history_obj_ctc.history:\n        plt.figure(figsize=(12, 5))\n        plt.subplot(1, 2, 1); plt.plot(history_obj_ctc.history['loss'], label='Train Loss')\n        if valid_ds and 'val_loss' in history_obj_ctc.history: plt.plot(history_obj_ctc.history['val_loss'], label='Val Loss')\n        plt.title('CTC Loss'); plt.xlabel('Epoch'); plt.ylabel('Loss'); plt.legend(); plt.grid(True)\n        plt.subplot(1, 2, 2); plt.plot(history_obj_ctc.history['edit_dist'], label='Train Edit Dist')\n        if valid_ds and 'val_edit_dist' in history_obj_ctc.history: plt.plot(history_obj_ctc.history['val_edit_dist'], label='Val Edit Dist')\n        plt.title('CTC Edit Distance'); plt.xlabel('Epoch'); plt.ylabel('Edit Dist'); plt.legend(); plt.grid(True)\n        plt.tight_layout()\n        try: plt.savefig(\"ctc_training_history.png\"); print(\"CTC Training history plot saved to ctc_training_history.png\")\n        except Exception as e: print(f\"Could not save CTC training history plot: {e}\")\n    model_instance_ctc.save_weights(\"ctc_model_final.weights.h5\"); print(\"CTC Model weights saved.\")\nelse: print(\"Training dataset is not available. Cannot start CTC training.\")\nprint(\"Script finished.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T12:10:11.857179Z","iopub.execute_input":"2025-05-20T12:10:11.858076Z"}},"outputs":[],"execution_count":null}]}