{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":52950,"databundleVersionId":5973250,"sourceType":"competition"},{"sourceId":133726408,"sourceType":"kernelVersion"}],"dockerImageVersionId":30527,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Import the Libraries for the Processing","metadata":{}},{"cell_type":"code","source":"import gc\nimport json\nimport math\nimport pickle\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow_addons as tfa\n\nimport os\nimport json\nimport random\n\nfrom skimage.transform import resize\nfrom tensorflow import keras\nfrom tensorflow.keras import layers","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-15T10:29:10.58375Z","iopub.execute_input":"2023-08-15T10:29:10.584209Z","iopub.status.idle":"2023-08-15T10:29:10.595679Z","shell.execute_reply.started":"2023-08-15T10:29:10.584164Z","shell.execute_reply":"2023-08-15T10:29:10.594792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check for the GPU config to utilise\nprint(\"GPU Available:\", tf.config.list_physical_devices('GPU'))","metadata":{"execution":{"iopub.status.busy":"2023-08-15T10:29:10.597709Z","iopub.execute_input":"2023-08-15T10:29:10.598379Z","iopub.status.idle":"2023-08-15T10:29:10.609586Z","shell.execute_reply.started":"2023-08-15T10:29:10.598345Z","shell.execute_reply":"2023-08-15T10:29:10.608655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\nprint(\"Full train dataset shape is {}\".format(dataset_df.shape))\n\nprint(dataset_df.head())\nprint(dataset_df.info())","metadata":{"execution":{"iopub.status.busy":"2023-08-15T10:29:10.611083Z","iopub.execute_input":"2023-08-15T10:29:10.611784Z","iopub.status.idle":"2023-08-15T10:29:10.913249Z","shell.execute_reply.started":"2023-08-15T10:29:10.611752Z","shell.execute_reply":"2023-08-15T10:29:10.91224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#understanding the imabalance in the phrases\nprint(dataset_df['phrase'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2023-08-15T10:29:10.918868Z","iopub.execute_input":"2023-08-15T10:29:10.921542Z","iopub.status.idle":"2023-08-15T10:29:10.976514Z","shell.execute_reply.started":"2023-08-15T10:29:10.921478Z","shell.execute_reply":"2023-08-15T10:29:10.975325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Enabling the GPU Processing and preprocessing the Data","metadata":{}},{"cell_type":"code","source":"pad_token = 'P'\nstart_token = 'S'\nend_token = 'E'\npad_token_idx = 59\nstart_token_idx = 60\nend_token_idx = 61\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\nchar_to_num[pad_token] = pad_token_idx\nchar_to_num[start_token] = start_token_idx\nchar_to_num[end_token] = end_token_idx\nnum_to_char = {j:i for i,j in char_to_num.items()}\n\ninpdir = \"/kaggle/input/asl-fingerspelling\"\ndf = pd.read_csv(f'{inpdir}/train.csv')\n\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nRHAND_LBLS = [f'x_right_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)] + [f'z_right_hand_{i}' for i in range(21)]\nLHAND_LBLS = [ f'x_left_hand_{i}' for i in range(21)] + [ f'y_left_hand_{i}' for i in range(21)] + [ f'z_left_hand_{i}' for i in range(21)]\nPOSE_LBLS = [f'x_pose_{i}' for i in POSE] + [f'y_pose_{i}' for i in POSE] + [f'z_pose_{i}' for i in POSE]\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE]\n\nSEL_COLS = X + Y + Z\nFRAME_LEN = 128\n\nX_IDX = [i for i, col in enumerate(SEL_COLS)  if \"x_\" in col]\nY_IDX = [i for i, col in enumerate(SEL_COLS)  if \"y_\" in col]\nZ_IDX = [i for i, col in enumerate(SEL_COLS)  if \"z_\" in col]\n\nRHAND_IDX = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col]\nLHAND_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col]\nRPOSE_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE]\nLPOSE_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE]","metadata":{"execution":{"iopub.status.busy":"2023-08-15T10:29:10.980788Z","iopub.execute_input":"2023-08-15T10:29:10.983052Z","iopub.status.idle":"2023-08-15T10:29:11.127508Z","shell.execute_reply.started":"2023-08-15T10:29:10.983015Z","shell.execute_reply":"2023-08-15T10:29:11.126519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_pad(x):\n    if tf.shape(x)[0] < FRAME_LEN:\n        x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]))\n    else:\n        x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n    return x\n\ndef pre_process(x):\n    rhand = tf.gather(x, RHAND_IDX, axis=1)\n    lhand = tf.gather(x, LHAND_IDX, axis=1)\n    rpose = tf.gather(x, RPOSE_IDX, axis=1)\n    lpose = tf.gather(x, LPOSE_IDX, axis=1)\n    \n    rnan_idx = tf.reduce_any(tf.math.is_nan(rhand), axis=1)\n    lnan_idx = tf.reduce_any(tf.math.is_nan(lhand), axis=1)\n    \n    rnans = tf.math.count_nonzero(rnan_idx)\n    lnans = tf.math.count_nonzero(lnan_idx)\n    \n    # For dominant hand\n    if rnans > lnans:\n        hand = lhand\n        pose = lpose\n        \n        hand_x = hand[:, 0*(len(LHAND_IDX)//3) : 1*(len(LHAND_IDX)//3)]\n        hand_y = hand[:, 1*(len(LHAND_IDX)//3) : 2*(len(LHAND_IDX)//3)]\n        hand_z = hand[:, 2*(len(LHAND_IDX)//3) : 3*(len(LHAND_IDX)//3)]\n        hand = tf.concat([1-hand_x, hand_y, hand_z], axis=1)\n        \n        pose_x = pose[:, 0*(len(LPOSE_IDX)//3) : 1*(len(LPOSE_IDX)//3)]\n        pose_y = pose[:, 1*(len(LPOSE_IDX)//3) : 2*(len(LPOSE_IDX)//3)]\n        pose_z = pose[:, 2*(len(LPOSE_IDX)//3) : 3*(len(LPOSE_IDX)//3)]\n        pose = tf.concat([1-pose_x, pose_y, pose_z], axis=1)\n    else:\n        hand = rhand\n        pose = rpose\n    \n    hand_x = hand[:, 0*(len(LHAND_IDX)//3) : 1*(len(LHAND_IDX)//3)]\n    hand_y = hand[:, 1*(len(LHAND_IDX)//3) : 2*(len(LHAND_IDX)//3)]\n    hand_z = hand[:, 2*(len(LHAND_IDX)//3) : 3*(len(LHAND_IDX)//3)]\n    hand = tf.concat([hand_x[..., tf.newaxis], hand_y[..., tf.newaxis], hand_z[..., tf.newaxis]], axis=-1)\n    \n    mean = tf.math.reduce_mean(hand, axis=1)[:, tf.newaxis, :]\n    std = tf.math.reduce_std(hand, axis=1)[:, tf.newaxis, :]\n    hand = (hand - mean) / std\n\n    pose_x = pose[:, 0*(len(LPOSE_IDX)//3) : 1*(len(LPOSE_IDX)//3)]\n    pose_y = pose[:, 1*(len(LPOSE_IDX)//3) : 2*(len(LPOSE_IDX)//3)]\n    pose_z = pose[:, 2*(len(LPOSE_IDX)//3) : 3*(len(LPOSE_IDX)//3)]\n    pose = tf.concat([pose_x[..., tf.newaxis], pose_y[..., tf.newaxis], pose_z[..., tf.newaxis]], axis=-1)\n    \n    x = tf.concat([hand, pose], axis=1)\n    x = resize_pad(x)\n    \n    x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n    x = x[None]\n    x = tf.reshape(x, (FRAME_LEN, len(LHAND_IDX) + len(LPOSE_IDX)))\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-08-15T10:29:11.132198Z","iopub.execute_input":"2023-08-15T10:29:11.134383Z","iopub.status.idle":"2023-08-15T10:29:11.162844Z","shell.execute_reply.started":"2023-08-15T10:29:11.134348Z","shell.execute_reply":"2023-08-15T10:29:11.161432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table = tf.lookup.StaticHashTable(\n    initializer=tf.lookup.KeyValueTensorInitializer(\n        keys=list(char_to_num.keys()),\n        values=list(char_to_num.values()),\n    ),\n    default_value=tf.constant(-1),\n    name=\"class_weight\"\n)\n\ndef preprocess_fn(landmarks, phrase):\n    phrase = start_token + phrase + end_token\n    phrase = tf.strings.bytes_split(phrase)\n    phrase = table.lookup(phrase)\n    phrase = tf.pad(phrase, paddings=[[0, 64 - tf.shape(phrase)[0]]], mode = 'CONSTANT',\n                    constant_values = pad_token_idx)\n    return pre_process(landmarks), phrase\n\ndef decode_fn(record_bytes):\n    schema = {COL: tf.io.VarLenFeature(dtype=tf.float32) for COL in SEL_COLS}\n    schema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n    features = tf.io.parse_single_example(record_bytes, schema)\n    phrase = features[\"phrase\"]\n    landmarks = ([tf.sparse.to_dense(features[COL]) for COL in SEL_COLS])\n    landmarks = tf.transpose(landmarks)\n    \n    return landmarks, phrase\n\ninpdir = \"/kaggle/input/aslfr-parquets-to-tfrecords-cleaned\"\ntffiles = df.file_id.map(lambda x: f'{inpdir}/tfds/{x}.tfrecord').unique()\n\nbatch_size = 32\nval_len = int(0.05 * len(tffiles))\n\ntrain_ds = tf.data.TFRecordDataset(tffiles[val_len:]).map(decode_fn).map(preprocess_fn).shuffle(30000, reshuffle_each_iteration=True).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)\nvalid_ds = tf.data.TFRecordDataset(tffiles[:val_len]).map(decode_fn).map(preprocess_fn).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-08-15T10:29:11.167212Z","iopub.execute_input":"2023-08-15T10:29:11.16755Z","iopub.status.idle":"2023-08-15T10:29:13.530925Z","shell.execute_reply.started":"2023-08-15T10:29:11.167523Z","shell.execute_reply":"2023-08-15T10:29:13.529797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(valid_ds)","metadata":{"execution":{"iopub.status.busy":"2023-08-15T10:29:13.535622Z","iopub.execute_input":"2023-08-15T10:29:13.537941Z","iopub.status.idle":"2023-08-15T10:29:13.548365Z","shell.execute_reply.started":"2023-08-15T10:29:13.537903Z","shell.execute_reply":"2023-08-15T10:29:13.547246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras_tuner as kt\nfrom tensorflow.keras import layers, regularizers\nfrom tensorflow.keras.callbacks import LearningRateScheduler","metadata":{"execution":{"iopub.status.busy":"2023-08-15T10:29:13.553154Z","iopub.execute_input":"2023-08-15T10:29:13.553751Z","iopub.status.idle":"2023-08-15T10:29:13.561392Z","shell.execute_reply.started":"2023-08-15T10:29:13.553694Z","shell.execute_reply":"2023-08-15T10:29:13.560471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TokenEmbedding(layers.Layer):\n    def __init__(self, num_vocab=1000, maxlen=100, num_hid=64):\n        super().__init__()\n        self.emb = tf.keras.layers.Embedding(num_vocab, num_hid)\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=num_hid)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        x = self.emb(x)\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        return x + positions\n\n\nclass LandmarkEmbedding(layers.Layer):\n    def __init__(self, num_hid=64, maxlen=100):\n        super().__init__()\n        self.conv1 = tf.keras.layers.Conv1D(\n            num_hid, 11, strides=2, padding=\"same\"\n        )\n        self.batchnorm1 = layers.BatchNormalization()\n        self.conv2 = tf.keras.layers.Conv1D(\n            num_hid, 7, strides=2, padding=\"same\"\n        )\n        self.batchnorm2 = layers.BatchNormalization()\n        self.conv3 = tf.keras.layers.Conv1D(\n            num_hid, 5, strides=2, padding=\"same\"\n        )\n        self.batchnorm3 = layers.BatchNormalization()\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=num_hid)\n\n    def call(self, x):\n        x = self.conv1(x)\n        x = self.batchnorm1(x)\n        x = tf.nn.relu(x)\n\n        x = self.conv2(x)\n        x = self.batchnorm2(x)\n        x = tf.nn.relu(x)\n\n        x = self.conv3(x)\n        x = self.batchnorm3(x)\n        x = tf.nn.relu(x)\n\n        return x\n    \nclass TransformerEncoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, rate=0.1):\n        super().__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.dropout1 = layers.Dropout(rate)\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dropout(rate),\n                layers.Dense(embed_dim),\n            ]\n        )\n        self.dropout2 = layers.Dropout(rate)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        \n        # Additional layers\n        self.att_2 = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.dropout3 = layers.Dropout(rate)\n        self.layernorm3 = layers.LayerNormalization(epsilon=1e-6)\n        self.ffn_2 = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dropout(rate),\n                layers.Dense(embed_dim),\n            ]\n        )\n        self.dropout4 = layers.Dropout(rate)\n        self.layernorm4 = layers.LayerNormalization(epsilon=1e-6)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        attn_output = self.layernorm1(inputs + attn_output)\n\n        ffn_output = self.ffn(attn_output)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        ffn_output = self.layernorm2(attn_output + ffn_output)\n        \n        # Additional layers\n        attn_output_2 = self.att_2(ffn_output, ffn_output)\n        attn_output_2 = self.dropout3(attn_output_2, training=training)\n        attn_output_2 = self.layernorm3(ffn_output + attn_output_2)\n\n        ffn_output_2 = self.ffn_2(attn_output_2)\n        ffn_output_2 = self.dropout4(ffn_output_2, training=training)\n        return self.layernorm4(attn_output_2 + ffn_output_2)\n\n\nclass TransformerDecoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, dropout_rate=0.1):\n        super().__init__()\n        self.self_att = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=embed_dim\n        )\n        self.enc_att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.dropout1 = layers.Dropout(dropout_rate)\n        self.dropout2 = layers.Dropout(dropout_rate)\n        self.dropout3 = layers.Dropout(dropout_rate)\n        self.dropout4 = layers.Dropout(dropout_rate)\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm4 = layers.LayerNormalization(epsilon=1e-6)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dropout(dropout_rate),\n                layers.Dense(embed_dim),\n            ]\n        )\n        self.ffn_2 = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dropout(dropout_rate),\n                layers.Dense(embed_dim),\n            ]\n        )\n        self.batchnorm1 = layers.BatchNormalization()\n        self.batchnorm2 = layers.BatchNormalization()\n        self.batchnorm3 = layers.BatchNormalization()\n        self.batchnorm4 = layers.BatchNormalization()\n\n    def causal_attention_mask(self, batch_size, n_dest, n_src, dtype):\n        \"\"\"Masks the upper half of the dot product matrix in self attention.\n        \n        This prevents the flow of information from future tokens to the current token.\n        1's in the lower triangle, counting from the lower right corner.\n        \"\"\"\n        i = tf.range(n_dest)[:, None]\n        j = tf.range(n_src)\n        m = i >= j - n_src + n_dest\n        mask = tf.cast(m, dtype)\n        mask = tf.reshape(mask, [1, n_dest, n_src])\n        mult = tf.concat(\n            [batch_size[..., tf.newaxis], tf.constant([1, 1], dtype=tf.int32)], 0\n        )\n        return tf.tile(mask, mult)\n\n    def call(self, enc_out, target, training):\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        seq_len = input_shape[1]\n        causal_mask = self.causal_attention_mask(batch_size, seq_len, seq_len, tf.bool)\n\n        target_att = self.self_att(target, target, attention_mask=causal_mask)\n        target_att = self.dropout1(target_att, training=training)\n        target_norm = self.layernorm1(target + target_att)\n\n        enc_out_att = self.enc_att(target_norm, enc_out)\n        enc_out_att = self.dropout2(enc_out_att, training=training)\n        enc_out_norm = self.layernorm2(target_norm + enc_out_att)\n\n        ffn_out = self.ffn(enc_out_norm)\n        ffn_out = self.dropout3(ffn_out, training=training)\n        \n        ffn2_out = self.ffn_2(ffn_out)\n        ffn2_out = self.dropout4(ffn2_out, training=training)\n        \n        return self.layernorm4(enc_out_norm + ffn2_out)\n\n\n    \nclass Transformer(keras.Model):\n    def __init__(\n        self,\n        num_hid=64,\n        num_head=2,\n        num_feed_forward=128,\n        source_maxlen=100,\n        target_maxlen=100,\n        num_layers_enc=4,\n        num_layers_dec=1,\n        num_classes=60,\n        \n    ):\n        super().__init__()\n        self.loss_metric = keras.metrics.Mean(name=\"loss\")\n        self.acc_metric = keras.metrics.Mean(name=\"edit_dist\")\n        self.num_layers_enc = num_layers_enc\n        self.num_layers_dec = num_layers_dec\n        self.target_maxlen = target_maxlen\n        self.num_classes = num_classes\n\n        self.enc_input = LandmarkEmbedding(num_hid=num_hid, maxlen=source_maxlen)\n        self.dec_input = TokenEmbedding(\n            num_vocab=num_classes, maxlen=target_maxlen, num_hid=num_hid\n        )\n\n        self.encoder = keras.Sequential(\n            [self.enc_input]\n            + [\n                TransformerEncoder(num_hid, num_head, num_feed_forward)\n                for _ in range(num_layers_enc)\n            ]\n        )\n\n        for i in range(num_layers_dec):\n            setattr(\n                self,\n                f\"dec_layer_{i}\",\n                TransformerDecoder(num_hid, num_head, num_feed_forward),\n            )\n\n        self.classifier = layers.Dense(num_classes)\n\n    def decode(self, enc_out, target, training):\n        y = self.dec_input(target)\n        for i in range(self.num_layers_dec):\n            y = getattr(self, f\"dec_layer_{i}\")(enc_out, y, training)\n        return y\n\n    def call(self, inputs, training):\n        source = inputs[0]\n        target = inputs[1]\n        x = self.encoder(source, training)\n        y = self.decode(x, target, training)\n        return self.classifier(y)\n\n    @property\n    def metrics(self):\n        return [self.loss_metric]\n\n    def train_step(self, batch):\n        \"\"\"Processes one batch inside model.fit().\"\"\"\n        source = batch[0]\n        target = batch[1]\n\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        \n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        with tf.GradientTape() as tape:\n            preds = self([source, dec_input])\n            one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n            mask = tf.math.logical_not(tf.math.equal(dec_target, pad_token_idx))\n            loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        trainable_vars = self.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n        # Computes the Levenshtein distance between sequences since the evaluation\n        # metric for this contest is the normalized total levenshtein distance.\n        edit_dist = tf.edit_distance(tf.sparse.from_dense(target), \n                                     tf.sparse.from_dense(tf.cast(tf.argmax(preds, axis=1), tf.int32)))\n        edit_dist = tf.reduce_mean(edit_dist)\n        self.acc_metric.update_state(edit_dist)\n        self.loss_metric.update_state(loss)\n        return {\"loss\": self.loss_metric.result(), \"edit_dist\": self.acc_metric.result()}\n\n    def test_step(self, batch):        \n        source = batch[0]\n        target = batch[1]\n\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        \n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        \n        with tf.GradientTape() as tape:\n            preds = self([source, dec_input], training=True)\n            one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n            mask = tf.math.logical_not(tf.math.equal(dec_target, pad_token_idx))\n            loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n\n        trainable_vars = self.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n        edit_dist = tf.edit_distance(tf.sparse.from_dense(dec_target), \n                                     tf.sparse.from_dense(tf.cast(tf.argmax(preds, axis=1), tf.int32)))\n        edit_dist = tf.reduce_mean(edit_dist)\n        self.acc_metric.update_state(edit_dist)\n        self.loss_metric.update_state(loss)\n        return {\"loss\": self.loss_metric.result(), \"edit_dist\": self.acc_metric.result()}\n\n\n    def generate(self, source, target_start_token_idx):\n        \"\"\"Performs inference over one batch of inputs using greedy decoding.\"\"\"\n        bs = tf.shape(source)[0]\n        enc = self.encoder(source, training = False)\n        dec_input = tf.ones((bs, 1), dtype=tf.int32) * target_start_token_idx\n        dec_logits = []\n        for i in range(self.target_maxlen - 1):\n            dec_out = self.decode(enc, dec_input, training = False)\n            logits = self.classifier(dec_out)\n            logits = tf.argmax(logits, axis=-1, output_type=tf.int32)\n            last_logit = logits[:, -1][..., tf.newaxis]\n            dec_logits.append(last_logit)\n            dec_input = tf.concat([dec_input, last_logit], axis=-1)\n        return dec_input\n    \nclass DisplayOutputs(keras.callbacks.Callback):\n    def __init__(\n        self, batch, idx_to_token, target_start_token_idx=60, target_end_token_idx=61\n    ):\n        \"\"\"Displays a batch of outputs after every 4 epoch\n\n        Args:\n            batch: A test batch\n            idx_to_token: A List containing the vocabulary tokens corresponding to their indices\n            target_start_token_idx: A start token index in the target vocabulary\n            target_end_token_idx: An end token index in the target vocabulary\n        \"\"\"\n        self.batch = batch\n        self.target_start_token_idx = target_start_token_idx\n        self.target_end_token_idx = target_end_token_idx\n        self.idx_to_char = idx_to_token\n\n    def on_epoch_end(self, epoch, logs=None):\n        if epoch % 4 != 0:\n            return\n        source = self.batch[0]\n        target = self.batch[1].numpy()\n        bs = tf.shape(source)[0]\n        preds = self.model.generate(source, self.target_start_token_idx)\n        preds = preds.numpy()\n        for i in range(bs):\n            target_text = \"\".join([self.idx_to_char[_] for _ in target[i, :]])\n            prediction = \"\"\n            for idx in preds[i, :]:\n                prediction += self.idx_to_char[idx]\n                if idx == self.target_end_token_idx:\n                    break\n            print(f\"target:     {target_text.replace('-','')}\")\n            print(f\"prediction: {prediction}\\n\")\n            \n# Modified code with enhancements\nbatch = next(iter(valid_ds))\nidx_to_char = list(char_to_num.keys())\n\ndisplay_cb = DisplayOutputs(\n    batch, idx_to_char, target_start_token_idx=char_to_num['S'], target_end_token_idx=char_to_num['E']\n)\n\nmodel = Transformer(\n    num_hid=200,\n    num_head=4,\n    num_feed_forward=400,\n    source_maxlen = FRAME_LEN,\n    target_maxlen=64,\n    num_layers_enc=2,\n    num_layers_dec=1,\n    num_classes=62\n)\n\n# Use a smaller learning rate and longer decay steps for better convergence\ninitial_learning_rate = 0.00005\nlr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n    initial_learning_rate,\n    decay_steps=5000,\n    decay_rate=0.96,\n    staircase=True\n)\n\noptimizer = keras.optimizers.Adam(lr_schedule, clipvalue=0.5)\n\nloss_fn = tf.keras.losses.CategoricalCrossentropy(\n    from_logits=True, label_smoothing=0.1,\n)\n\nmodel.compile(optimizer=optimizer, loss=loss_fn)\n\n# Train the model with early stopping and modified hyperparameters\nearly_stopping = keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True)\n\nhistory = model.fit(train_ds, validation_data=valid_ds, callbacks=[display_cb, early_stopping], epochs=55)","metadata":{"execution":{"iopub.status.busy":"2023-08-15T10:29:13.570482Z","iopub.execute_input":"2023-08-15T10:29:13.572906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n1520/1520 [==============================] - 469s 236ms/step - loss: 0.7645 - edit_dist: 1.1165 - val_loss: 0.9847 - val_edit_dist: 1.0981\n\"\"\"\n!pip install tflite-runtime","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model.summary())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.legend(['training loss', 'val_loss'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batches = [batch for batch in valid_ds]\n\npreds_list = []\nground_truth_list = []\n\nfor batch in batches[:1]:\n    source = batch[0]\n    target = batch[1].numpy()\n    bs = tf.shape(source)[0]\n    preds = model.generate(source, start_token_idx)\n    preds = preds.numpy()\n\n    for i in range(bs):\n        target_text = \"\".join([idx_to_char[_] for _ in target[i, :]])\n        ground_truth_list.append(target_text.replace('P', ''))\n        prediction = \"\"\n        for idx in preds[i, :]:\n            prediction += idx_to_char[idx]\n            if idx == end_token_idx:\n                break\n        preds_list.append(prediction)\n\nfor i in range(10):\n    print(ground_truth_list[i])\n    print(preds_list[i])\n    print('\\n~~~\\n')","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import Levenshtein as lev","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ground_truth_processed = [ground_truth_list[i][1:-1] for i in range(len(ground_truth_list))]\npreds_list_processed = [preds_list[i][1:-1] for i in range(len(preds_list))]\nlev_dist = [lev.distance(ground_truth_processed[i], preds_list_processed[i]) \n            for i in range(len(preds_list_processed))]\nN = [len(phrase) for phrase in ground_truth_processed]\n\nprint('Validation score: '+str((np.sum(N) - np.sum(lev_dist))/np.sum(N)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" class TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n        self.target_start_token_idx = start_token_idx\n        self.target_end_token_idx = end_token_idx\n        # Load the feature generation and main models\n        self.model = model\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(SEL_COLS)], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs, training=False):\n        # Preprocess Data\n        x = tf.cast(inputs, tf.float32)\n        x = x[None]\n        x = tf.cond(tf.shape(x)[1] == 0, lambda: tf.zeros((1, 1, len(SEL_COLS))), lambda: tf.identity(x))\n        x = x[0]\n        x = pre_process(x)\n        x = x[None]\n        x = self.model.generate(x, self.target_start_token_idx)\n        x = x[0]\n        idx = tf.argmax(tf.cast(tf.equal(x, self.target_end_token_idx), tf.int32))\n        idx = tf.where(tf.math.less(idx, 1), tf.constant(2, dtype=tf.int64), idx)\n        x = x[1:idx]\n        x = tf.one_hot(x, 59)\n        return {'outputs': x}\n    \ntflitemodel_base = TFLiteModel(model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights(\"model.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]#, tf.lite.OpsSet.SELECT_TF_OPS]\ntflite_model = keras_model_converter.convert()\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \ninfargs = {\"selected_columns\" : SEL_COLS}\n\nwith open('inference_args.json', \"w\") as json_file:\n    json.dump(infargs, json_file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip  './model.tflite' './inference_args.json'\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tf.lite.Interpreter(\"model.tflite\")\n\nREQUIRED_SIGNATURE = \"serving_default\"\nREQUIRED_OUTPUT = \"outputs\"\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\nif REQUIRED_SIGNATURE not in found_signatures:\n    raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=batch[0][0])\nprediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\nprint(prediction_str)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport json\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport Levenshtein as Lev\nimport tensorflow as tf\n\n\nSEL_FEATURES = json.load(open('/kaggle/working/inference_args.json'))['selected_columns']\n\ndef load_relevant_data_subset(pq_path):\n        return pd.read_parquet(pq_path, columns=SEL_FEATURES) #selected_columns)\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\n\ndf = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n\nidx = 0\nsample = df.loc[idx]\nloaded = load_relevant_data_subset('/kaggle/input/asl-fingerspelling/' + sample['path'])\nloaded = loaded[loaded.index==sample['sequence_id']].values\nprint(loaded.shape)\nframes = loaded\n\ndef wer__(s1, s2):\n    w1 = len(s1.split())\n    lvd = Lev.distance(s1, s2)\n    return lvd / w1\n\n# import tflite_runtime.interpreter as tflite\n# interpreter = tflite.Interpreter('model.tflite')\nfound_signatures = list(interpreter.get_signature_list().keys())\n\nREQUIRED_SIGNATURE = 'serving_default'\nREQUIRED_OUTPUT = 'outputs'\nif REQUIRED_SIGNATURE not in found_signatures:\n    raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput_lite = prediction_fn(inputs=frames)\nprediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output_lite[REQUIRED_OUTPUT], axis=1)])\nprint(prediction_str)\n\n\nst = time.time()\ncnt = 0\ntotal = 100 #len(df)\nmodel_time = 0\n\nlevs = []\n\nfor i in tqdm(range(len(df.iloc[:total]))):\n    sample = df.loc[i]\n    loaded = load_relevant_data_subset('/kaggle/input/asl-fingerspelling/' + sample['path'])\n    loaded = loaded[loaded.index==sample['sequence_id']].values\n\n    md_st = time.time()\n    output_ = prediction_fn(inputs=loaded)\n    model_time += time.time() - md_st\n\n\n    prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output_[REQUIRED_OUTPUT], axis=1)])\n    cur_lev = wer__(sample['phrase'], prediction_str) \n    #print(sample['phrase'], '|', prediction_str, '|', cur_lev)\n    #print()\n\n    levs.append(cur_lev)\n\nprint(f'WER: {np.mean(levs):.5f}')\nprint(f'Mean time: {(time.time() - st)/total:.7f}')\nprint(f'Mean time only infer: {model_time/total:.7f}')\n\n#74/100 [02:23<00:51, 1.93s/it]\n#24/100 [00:17<00:55, 1.17it/s]\n#74/100 [01:27<00:30, 1.11s/it]","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}