{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This work is based on Rohith Ingilela's [notebook](https://www.kaggle.com/code/irohith/aslfr-transformer), KIM CHANG HYUN's [notebook](https://www.kaggle.com/code/changcom/aslfr-a-simple-transformer) and Gusthema's [notebook](https://www.kaggle.com/code/gusthema/asl-fingerspelling-recognition-w-tensorflow).","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom skimage.transform import resize\nfrom sklearn.model_selection import train_test_split\nimport json\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport Levenshtein as lev\nimport os\nimport gc","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:47:20.792977Z","iopub.execute_input":"2023-07-07T14:47:20.793335Z","iopub.status.idle":"2023-07-07T14:47:29.782174Z","shell.execute_reply.started":"2023-07-07T14:47:20.793306Z","shell.execute_reply":"2023-07-07T14:47:29.781195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = np.random.randint(50000)\nprint(seed)\nseed = 35567\ntf.random.set_seed(seed)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:47:29.807076Z","iopub.execute_input":"2023-07-07T14:47:29.80765Z","iopub.status.idle":"2023-07-07T14:47:29.817419Z","shell.execute_reply.started":"2023-07-07T14:47:29.807615Z","shell.execute_reply":"2023-07-07T14:47:29.816463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data loading","metadata":{}},{"cell_type":"markdown","source":"#### Here I use new tokens for padding, start and end of sentences. (Capitals are good since the original phrases have only lower case letters, besides numbers and various signs).","metadata":{}},{"cell_type":"code","source":"pad_token = 'P'\nstart_token = 'S'\nend_token = 'E'\npad_token_idx = 59\nstart_token_idx = 60\nend_token_idx = 61","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:47:32.743082Z","iopub.execute_input":"2023-07-07T14:47:32.743445Z","iopub.status.idle":"2023-07-07T14:47:32.748852Z","shell.execute_reply.started":"2023-07-07T14:47:32.743416Z","shell.execute_reply":"2023-07-07T14:47:32.747591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\nchar_to_num[pad_token] = pad_token_idx\nchar_to_num[start_token] = start_token_idx\nchar_to_num[end_token] = end_token_idx\nnum_to_char = {j:i for i,j in char_to_num.items()}\n\ninpdir = \"/kaggle/input/asl-fingerspelling\"\ndf = pd.read_csv(f'{inpdir}/train.csv')\n\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nRHAND_LBLS = [f'x_right_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)] + [f'z_right_hand_{i}' for i in range(21)]\nLHAND_LBLS = [ f'x_left_hand_{i}' for i in range(21)] + [ f'y_left_hand_{i}' for i in range(21)] + [ f'z_left_hand_{i}' for i in range(21)]\nPOSE_LBLS = [f'x_pose_{i}' for i in POSE] + [f'y_pose_{i}' for i in POSE] + [f'z_pose_{i}' for i in POSE]\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE]\n\nSEL_COLS = X + Y + Z\nFRAME_LEN = 128\n\nX_IDX = [i for i, col in enumerate(SEL_COLS)  if \"x_\" in col]\nY_IDX = [i for i, col in enumerate(SEL_COLS)  if \"y_\" in col]\nZ_IDX = [i for i, col in enumerate(SEL_COLS)  if \"z_\" in col]\n\nRHAND_IDX = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col]\nLHAND_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col]\nRPOSE_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE]\nLPOSE_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE]","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:47:39.027544Z","iopub.execute_input":"2023-07-07T14:47:39.0279Z","iopub.status.idle":"2023-07-07T14:47:39.184386Z","shell.execute_reply.started":"2023-07-07T14:47:39.027871Z","shell.execute_reply":"2023-07-07T14:47:39.183433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"first = pd.read_parquet(f'{inpdir}/train_landmarks/5414471.parquet')\nfirst.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:47:44.113859Z","iopub.execute_input":"2023-07-07T14:47:44.114307Z","iopub.status.idle":"2023-07-07T14:48:00.266796Z","shell.execute_reply.started":"2023-07-07T14:47:44.114273Z","shell.execute_reply":"2023-07-07T14:48:00.265876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"first[first.index == 1816825349].x_face_0.hist()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:48:00.268754Z","iopub.execute_input":"2023-07-07T14:48:00.269114Z","iopub.status.idle":"2023-07-07T14:48:00.610863Z","shell.execute_reply.started":"2023-07-07T14:48:00.269082Z","shell.execute_reply":"2023-07-07T14:48:00.609975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\npattern =re.compile('[^a-zA-Z]')\nset([pattern.sub('',col) for col in first.columns])\n","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:48:00.612374Z","iopub.execute_input":"2023-07-07T14:48:00.612981Z","iopub.status.idle":"2023-07-07T14:48:00.62468Z","shell.execute_reply.started":"2023-07-07T14:48:00.612946Z","shell.execute_reply":"2023-07-07T14:48:00.623622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[col for col in first.columns if 'x_right_hand_' in col]\n","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:48:00.627219Z","iopub.execute_input":"2023-07-07T14:48:00.627714Z","iopub.status.idle":"2023-07-07T14:48:00.638141Z","shell.execute_reply.started":"2023-07-07T14:48:00.627682Z","shell.execute_reply":"2023-07-07T14:48:00.637034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_pad(x):\n    if tf.shape(x)[0] < FRAME_LEN:\n        x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]))\n    else:\n        x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n    return x\n\ndef pre_process(x):\n    rhand = tf.gather(x, RHAND_IDX, axis=1)\n    lhand = tf.gather(x, LHAND_IDX, axis=1)\n    rpose = tf.gather(x, RPOSE_IDX, axis=1)\n    lpose = tf.gather(x, LPOSE_IDX, axis=1)\n    \n    rnan_idx = tf.reduce_any(tf.math.is_nan(rhand), axis=1)\n    lnan_idx = tf.reduce_any(tf.math.is_nan(lhand), axis=1)\n    \n    rnans = tf.math.count_nonzero(rnan_idx)\n    lnans = tf.math.count_nonzero(lnan_idx)\n    \n    # For dominant hand\n    if rnans > lnans:\n        hand = lhand\n        pose = lpose\n        \n        hand_x = hand[:, 0*(len(LHAND_IDX)//3) : 1*(len(LHAND_IDX)//3)]\n        hand_y = hand[:, 1*(len(LHAND_IDX)//3) : 2*(len(LHAND_IDX)//3)]\n        hand_z = hand[:, 2*(len(LHAND_IDX)//3) : 3*(len(LHAND_IDX)//3)]\n        hand = tf.concat([1-hand_x, hand_y, hand_z], axis=1)\n        \n        pose_x = pose[:, 0*(len(LPOSE_IDX)//3) : 1*(len(LPOSE_IDX)//3)]\n        pose_y = pose[:, 1*(len(LPOSE_IDX)//3) : 2*(len(LPOSE_IDX)//3)]\n        pose_z = pose[:, 2*(len(LPOSE_IDX)//3) : 3*(len(LPOSE_IDX)//3)]\n        pose = tf.concat([1-pose_x, pose_y, pose_z], axis=1)\n    else:\n        hand = rhand\n        pose = rpose\n    \n    hand_x = hand[:, 0*(len(LHAND_IDX)//3) : 1*(len(LHAND_IDX)//3)]\n    hand_y = hand[:, 1*(len(LHAND_IDX)//3) : 2*(len(LHAND_IDX)//3)]\n    hand_z = hand[:, 2*(len(LHAND_IDX)//3) : 3*(len(LHAND_IDX)//3)]\n    hand = tf.concat([hand_x[..., tf.newaxis], hand_y[..., tf.newaxis], hand_z[..., tf.newaxis]], axis=-1)\n    \n    mean = tf.math.reduce_mean(hand, axis=1)[:, tf.newaxis, :]\n    std = tf.math.reduce_std(hand, axis=1)[:, tf.newaxis, :]\n    hand = (hand - mean) / std\n\n    pose_x = pose[:, 0*(len(LPOSE_IDX)//3) : 1*(len(LPOSE_IDX)//3)]\n    pose_y = pose[:, 1*(len(LPOSE_IDX)//3) : 2*(len(LPOSE_IDX)//3)]\n    pose_z = pose[:, 2*(len(LPOSE_IDX)//3) : 3*(len(LPOSE_IDX)//3)]\n    pose = tf.concat([pose_x[..., tf.newaxis], pose_y[..., tf.newaxis], pose_z[..., tf.newaxis]], axis=-1)\n    \n    x = tf.concat([hand, pose], axis=1)\n    x = resize_pad(x)\n    \n    x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n    x = tf.reshape(x, (FRAME_LEN, len(LHAND_IDX) + len(LPOSE_IDX)))\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:48:00.639694Z","iopub.execute_input":"2023-07-07T14:48:00.640239Z","iopub.status.idle":"2023-07-07T14:48:00.666654Z","shell.execute_reply.started":"2023-07-07T14:48:00.640206Z","shell.execute_reply":"2023-07-07T14:48:00.66595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table = tf.lookup.StaticHashTable(\n    initializer=tf.lookup.KeyValueTensorInitializer(\n        keys=list(char_to_num.keys()),\n        values=list(char_to_num.values()),\n    ),\n    default_value=tf.constant(-1),\n    name=\"class_weight\"\n)\n\ndef preprocess_fn(landmarks, phrase):\n    phrase = start_token + phrase + end_token\n    phrase = tf.strings.bytes_split(phrase)\n    phrase = table.lookup(phrase)\n    phrase = tf.pad(phrase, paddings=[[0, 64 - tf.shape(phrase)[0]]], mode = 'CONSTANT',\n                    constant_values = pad_token_idx)\n    return pre_process(landmarks), phrase\n\ndef decode_fn(record_bytes):\n    schema = {COL: tf.io.VarLenFeature(dtype=tf.float32) for COL in SEL_COLS}\n    schema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n    features = tf.io.parse_single_example(record_bytes, schema)\n    phrase = features[\"phrase\"]\n    landmarks = ([tf.sparse.to_dense(features[COL]) for COL in SEL_COLS])\n    landmarks = tf.transpose(landmarks)\n    \n    return landmarks, phrase\n\ninpdir = \"/kaggle/input/aslfr-parquets-to-tfrecords-cleaned\"\ntffiles = df.file_id.map(lambda x: f'{inpdir}/tfds/{x}.tfrecord').unique()\n\nbatch_size = 32\nval_len = int(0.05 * len(tffiles))\n\ntrain_dataset = tf.data.TFRecordDataset(tffiles[val_len:]).map(decode_fn).map(preprocess_fn).shuffle(30000, reshuffle_each_iteration=True).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)\nval_dataset = tf.data.TFRecordDataset(tffiles[:val_len]).map(decode_fn).map(preprocess_fn).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:48:00.667969Z","iopub.execute_input":"2023-07-07T14:48:00.668598Z","iopub.status.idle":"2023-07-07T14:48:05.393584Z","shell.execute_reply.started":"2023-07-07T14:48:00.668562Z","shell.execute_reply":"2023-07-07T14:48:05.392626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The model","metadata":{}},{"cell_type":"markdown","source":"### Define the Transformer Input Layers\nWhen processing landmark coordinate features for the encoder, we apply convolutional layers to downsample them and process local relationships.\n\nWe sum position embeddings and token embeddings when processing past target tokens for the decoder.","metadata":{}},{"cell_type":"code","source":"class TokenEmbedding(layers.Layer):\n    def __init__(self, num_vocab=1000, maxlen=100, num_hid=64):\n        super().__init__()\n        self.num_hid = num_hid\n        self.emb = tf.keras.layers.Embedding(num_vocab, num_hid)\n        #self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=num_hid)\n        '''\n        self.pos_emb = tf.math.divide(\n            self.positional_encoding(maxlen-1, num_hid),\n            tf.math.sqrt(tf.cast(num_hid, tf.float32)))\n        '''\n        self.pos_emb = self.positional_encoding(maxlen-1, num_hid)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        x = self.emb(x)\n        x = tf.math.multiply(x, tf.math.sqrt(tf.cast(self.num_hid, tf.float32)))\n        '''\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        return x + positions\n        '''\n        return x + self.pos_emb[:maxlen, :]\n    \n    def positional_encoding(self, maxlen, num_hid):\n        depth = num_hid/2\n        positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n        depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n        angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n        angle_rads = tf.linalg.matmul(positions, angle_rates)\n        pos_encoding = tf.concat(\n          [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n          axis=-1) \n        return pos_encoding\n\n\nclass LandmarkEmbedding(layers.Layer):\n    def __init__(self, num_hid=64, maxlen=100):\n        super().__init__()\n        self.conv1 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n        self.conv2 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n        self.conv3 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n        self.pos_emb = self.positional_encoding(maxlen, num_hid)\n        self.maxlen = maxlen\n        self.num_hid = num_hid\n\n    def call(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        \n        x = tf.math.multiply(x, tf.math.sqrt(tf.cast(self.num_hid, tf.float32)))\n        x = x + self.pos_emb\n        \n        return x\n    \n    def positional_encoding(self, maxlen, num_hid):\n        depth = num_hid/2\n        positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n        depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n        angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n        angle_rads = tf.linalg.matmul(positions, angle_rates)\n        pos_encoding = tf.concat(\n          [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n          axis=-1) \n        return pos_encoding","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:48:05.39517Z","iopub.execute_input":"2023-07-07T14:48:05.395522Z","iopub.status.idle":"2023-07-07T14:48:05.413605Z","shell.execute_reply.started":"2023-07-07T14:48:05.395491Z","shell.execute_reply":"2023-07-07T14:48:05.412502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Encoder layer for Transformer","metadata":{}},{"cell_type":"code","source":"class TransformerEncoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, rate=0.1):\n        super().__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dense(embed_dim),\n            ]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:48:05.416396Z","iopub.execute_input":"2023-07-07T14:48:05.416804Z","iopub.status.idle":"2023-07-07T14:48:05.4333Z","shell.execute_reply.started":"2023-07-07T14:48:05.416772Z","shell.execute_reply":"2023-07-07T14:48:05.43216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Decoder layer for Transformer","metadata":{}},{"cell_type":"code","source":"class TransformerDecoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, dropout_rate=0.1):\n        super().__init__()\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = layers.LayerNormalization(epsilon=1e-6)\n        self.self_att = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=embed_dim\n        )\n        self.enc_att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.self_dropout = layers.Dropout(0.5)\n        self.enc_dropout = layers.Dropout(0.1)\n        self.ffn_dropout = layers.Dropout(0.1)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dense(embed_dim),\n            ]\n        )\n\n    def causal_attention_mask(self, batch_size, n_dest, n_src, dtype):\n        \"\"\"Masks the upper half of the dot product matrix in self attention.\n\n        This prevents flow of information from future tokens to current token.\n        1's in the lower triangle, counting from the lower right corner.\n        \"\"\"\n        i = tf.range(n_dest)[:, None]\n        j = tf.range(n_src)\n        m = i >= j - n_src + n_dest\n        mask = tf.cast(m, dtype)\n        mask = tf.reshape(mask, [1, n_dest, n_src])\n        mult = tf.concat(\n            [batch_size[..., tf.newaxis], tf.constant([1, 1], dtype=tf.int32)], 0\n        )\n        return tf.tile(mask, mult)\n\n    def call(self, enc_out, target, training):\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        seq_len = input_shape[1]\n        causal_mask = self.causal_attention_mask(batch_size, seq_len, seq_len, tf.bool)\n        target_att = self.self_att(target, target, attention_mask=causal_mask)\n        target_norm = self.layernorm1(target + self.self_dropout(target_att, training = training))\n        enc_out = self.enc_att(target_norm, enc_out)\n        enc_out_norm = self.layernorm2(self.enc_dropout(enc_out, training = training) + target_norm)\n        ffn_out = self.ffn(enc_out_norm)\n        ffn_out_norm = self.layernorm3(enc_out_norm + self.ffn_dropout(ffn_out, training = training))\n        return ffn_out_norm","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:48:05.874376Z","iopub.execute_input":"2023-07-07T14:48:05.874715Z","iopub.status.idle":"2023-07-07T14:48:05.888009Z","shell.execute_reply.started":"2023-07-07T14:48:05.874688Z","shell.execute_reply":"2023-07-07T14:48:05.886847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This model takes landmark coordinates as inputs and predicts a sequence of characters. The target character sequence, which has been shifted to the left is provided as the input to the decoder during training. The decoder employs its own past predictions during inference to forecast the next token.\n\nThe Levenshtein Distance between sequences is used as the accuracy metric since the evaluation metric for this contest is the Normalized Total Levenshtein Distance.","metadata":{}},{"cell_type":"code","source":"class Transformer(keras.Model):\n    def __init__(\n        self,\n        num_hid=64,\n        num_head=2,\n        num_feed_forward=128,\n        source_maxlen=100,\n        target_maxlen=100,\n        num_layers_enc=4,\n        num_layers_dec=1,\n        num_classes=60,\n    ):\n        super().__init__()\n        self.loss_metric = keras.metrics.Mean(name=\"loss\")\n        self.num_layers_enc = num_layers_enc\n        self.num_layers_dec = num_layers_dec\n        self.target_maxlen = target_maxlen\n        self.num_classes = num_classes\n\n        self.enc_input = LandmarkEmbedding(num_hid=num_hid, maxlen=source_maxlen)\n        self.dec_input = TokenEmbedding(\n            num_vocab=num_classes, maxlen=target_maxlen, num_hid=num_hid\n        )\n\n        self.encoder = keras.Sequential(\n            [self.enc_input]\n            + [\n                TransformerEncoder(num_hid, num_head, num_feed_forward)\n                for _ in range(num_layers_enc)\n            ]\n        )\n\n        for i in range(num_layers_dec):\n            setattr(\n                self,\n                f\"dec_layer_{i}\",\n                TransformerDecoder(num_hid, num_head, num_feed_forward),\n            )\n\n        self.classifier = layers.Dense(num_classes)\n\n    def decode(self, enc_out, target, training):\n        y = self.dec_input(target)\n        for i in range(self.num_layers_dec):\n            y = getattr(self, f\"dec_layer_{i}\")(enc_out, y, training)\n        return y\n\n    def call(self, inputs, training):\n        source = inputs[0]\n        target = inputs[1]\n        x = self.encoder(source, training)\n        y = self.decode(x, target, training)\n        return self.classifier(y)\n\n    @property\n    def metrics(self):\n        return [self.loss_metric]\n\n    def train_step(self, batch):\n        \"\"\"Processes one batch inside model.fit().\"\"\"\n        source = batch[0]\n        target = batch[1]\n\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        \n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        with tf.GradientTape() as tape:\n            preds = self([source, dec_input])\n            one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n            mask = tf.math.logical_not(tf.math.equal(dec_target, pad_token_idx))\n            loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        trainable_vars = self.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n        self.loss_metric.update_state(loss)\n        return {\"loss\": self.loss_metric.result()}\n\n    def test_step(self, batch):        \n        source = batch[0]\n        target = batch[1]\n\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        \n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        preds = self([source, dec_input])\n        one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n        mask = tf.math.logical_not(tf.math.equal(dec_target, pad_token_idx))\n        loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        self.loss_metric.update_state(loss)\n        return {\"loss\": self.loss_metric.result()}\n\n    def generate(self, source, target_start_token_idx):\n        \"\"\"Performs inference over one batch of inputs using greedy decoding.\"\"\"\n        bs = tf.shape(source)[0]\n        enc = self.encoder(source, training = False)\n        dec_input = tf.ones((bs, 1), dtype=tf.int32) * target_start_token_idx\n        dec_logits = []\n        for i in range(self.target_maxlen - 1):\n            dec_out = self.decode(enc, dec_input, training = False)\n            logits = self.classifier(dec_out)\n            logits = tf.argmax(logits, axis=-1, output_type=tf.int32)\n            last_logit = logits[:, -1][..., tf.newaxis]\n            dec_logits.append(last_logit)\n            dec_input = tf.concat([dec_input, last_logit], axis=-1)\n        return dec_input","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:48:08.975266Z","iopub.execute_input":"2023-07-07T14:48:08.975627Z","iopub.status.idle":"2023-07-07T14:48:08.998498Z","shell.execute_reply.started":"2023-07-07T14:48:08.975598Z","shell.execute_reply":"2023-07-07T14:48:08.997504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following callback function is used to display predictions.","metadata":{}},{"cell_type":"code","source":"class DisplayOutputs(keras.callbacks.Callback):\n    def __init__(\n        self, batch, idx_to_token, target_start_token_idx=60, target_end_token_idx=61\n    ):\n        \"\"\"Displays a batch of outputs after every 4 epoch\n\n        Args:\n            batch: A test batch\n            idx_to_token: A List containing the vocabulary tokens corresponding to their indices\n            target_start_token_idx: A start token index in the target vocabulary\n            target_end_token_idx: An end token index in the target vocabulary\n        \"\"\"\n        self.batch = batch\n        self.target_start_token_idx = target_start_token_idx\n        self.target_end_token_idx = target_end_token_idx\n        self.idx_to_char = idx_to_token\n\n    def on_epoch_end(self, epoch, logs=None):\n        if epoch % 4 != 0:\n            return\n        source = self.batch[0]\n        target = self.batch[1].numpy()\n        bs = tf.shape(source)[0]\n        preds = self.model.generate(source, self.target_start_token_idx)\n        preds = preds.numpy()\n        for i in range(bs):\n            target_text = \"\".join([self.idx_to_char[_] for _ in target[i, :]])\n            prediction = \"\"\n            for idx in preds[i, :]:\n                prediction += self.idx_to_char[idx]\n                if idx == self.target_end_token_idx:\n                    break\n            print(f\"target:     {target_text.replace('-','')}\")\n            print(f\"prediction: {prediction}\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:48:12.95854Z","iopub.execute_input":"2023-07-07T14:48:12.959134Z","iopub.status.idle":"2023-07-07T14:48:12.989934Z","shell.execute_reply.started":"2023-07-07T14:48:12.95909Z","shell.execute_reply":"2023-07-07T14:48:12.988928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch = next(iter(val_dataset))\n\n# The vocabulary to convert predicted indices into characters\nidx_to_char = list(char_to_num.keys())\ndisplay_cb = DisplayOutputs(\n    batch, idx_to_char, target_start_token_idx=char_to_num['('], target_end_token_idx=char_to_num[')']\n)  # set the arguments as per vocabulary index for '<' and '>'\n\nmodel = Transformer(\n    num_hid=200,\n    num_head=4,\n    num_feed_forward=400,\n    source_maxlen = FRAME_LEN,\n    target_maxlen=64,\n    num_layers_enc=3,\n    num_layers_dec=2,\n    num_classes=62\n)\nloss_fn = tf.keras.losses.CategoricalCrossentropy(\n    from_logits=True, label_smoothing=0.1,\n)\n\n\noptimizer = keras.optimizers.Adam(0.0001)\nmodel.compile(optimizer=optimizer, loss=loss_fn)\n\nhistory = model.fit(train_dataset, validation_data=val_dataset, callbacks=[display_cb], epochs=13, verbose = 2)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T14:50:11.14451Z","iopub.execute_input":"2023-07-07T14:50:11.144866Z","iopub.status.idle":"2023-07-07T15:27:38.45749Z","shell.execute_reply.started":"2023-07-07T14:50:11.144839Z","shell.execute_reply":"2023-07-07T15:27:38.453873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model.summary())","metadata":{"execution":{"iopub.status.busy":"2023-07-07T15:27:39.006314Z","iopub.execute_input":"2023-07-07T15:27:39.006989Z","iopub.status.idle":"2023-07-07T15:27:39.059564Z","shell.execute_reply.started":"2023-07-07T15:27:39.006954Z","shell.execute_reply":"2023-07-07T15:27:39.058757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])","metadata":{"execution":{"iopub.status.busy":"2023-07-07T15:27:38.510234Z","iopub.execute_input":"2023-07-07T15:27:38.510581Z","iopub.status.idle":"2023-07-07T15:27:38.907154Z","shell.execute_reply.started":"2023-07-07T15:27:38.510549Z","shell.execute_reply":"2023-07-07T15:27:38.906236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation","metadata":{}},{"cell_type":"code","source":"batches = [batch for batch in val_dataset]\n\npreds_list = []\nground_truth_list = []\n\nfor batch in batches[:1]:\n    source = batch[0]\n    target = batch[1].numpy()\n    bs = tf.shape(source)[0]\n    preds = model.generate(source, start_token_idx)\n    preds = preds.numpy()\n\n    for i in range(bs):\n        target_text = \"\".join([idx_to_char[_] for _ in target[i, :]])\n        ground_truth_list.append(target_text.replace('P', ''))\n        prediction = \"\"\n        for idx in preds[i, :]:\n            prediction += idx_to_char[idx]\n            if idx == end_token_idx:\n                break\n        preds_list.append(prediction)\n\nfor i in range(10):\n    print(ground_truth_list[i])\n    print(preds_list[i])\n    print('\\n~~~\\n')","metadata":{"execution":{"iopub.status.busy":"2023-07-07T15:38:38.539974Z","iopub.execute_input":"2023-07-07T15:38:38.540997Z","iopub.status.idle":"2023-07-07T15:38:43.512127Z","shell.execute_reply.started":"2023-07-07T15:38:38.540951Z","shell.execute_reply":"2023-07-07T15:38:43.511033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ground_truth_processed = [ground_truth_list[i][1:-1] for i in range(len(ground_truth_list))]\npreds_list_processed = [preds_list[i][1:-1] for i in range(len(preds_list))]\nlev_dist = [lev.distance(ground_truth_processed[i], preds_list_processed[i]) \n            for i in range(len(preds_list_processed))]\nN = [len(phrase) for phrase in ground_truth_processed]\n\nprint('Validation score: '+str((np.sum(N) - np.sum(lev_dist))/np.sum(N)))","metadata":{"execution":{"iopub.status.busy":"2023-07-07T15:38:52.771133Z","iopub.execute_input":"2023-07-07T15:38:52.772032Z","iopub.status.idle":"2023-07-07T15:38:52.779676Z","shell.execute_reply.started":"2023-07-07T15:38:52.771997Z","shell.execute_reply":"2023-07-07T15:38:52.778716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TFLiteModel","metadata":{}},{"cell_type":"code","source":" class TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n        self.target_start_token_idx = start_token_idx\n        self.target_end_token_idx = end_token_idx\n        # Load the feature generation and main models\n        self.model = model\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(SEL_COLS)], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs, training=False):\n        # Preprocess Data\n        x = tf.cast(inputs, tf.float32)\n        x = x[None]\n        x = tf.cond(tf.shape(x)[1] == 0, lambda: tf.zeros((1, 1, len(SEL_COLS))), lambda: tf.identity(x))\n        x = x[0]\n        x = pre_process(x)\n        x = x[None]\n        x = self.model.generate(x, self.target_start_token_idx)\n        x = x[0]\n        idx = tf.argmax(tf.cast(tf.equal(x, self.target_end_token_idx), tf.int32))\n        idx = tf.where(tf.math.less(idx, 1), tf.constant(2, dtype=tf.int64), idx)\n        x = x[1:idx]\n        x = tf.one_hot(x, 59)\n        return {'outputs': x}\n    \ntflitemodel_base = TFLiteModel(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T15:39:05.54883Z","iopub.execute_input":"2023-07-07T15:39:05.549545Z","iopub.status.idle":"2023-07-07T15:39:05.562653Z","shell.execute_reply.started":"2023-07-07T15:39:05.549513Z","shell.execute_reply":"2023-07-07T15:39:05.561595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights(\"model.h5\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T15:39:09.248589Z","iopub.execute_input":"2023-07-07T15:39:09.248983Z","iopub.status.idle":"2023-07-07T15:39:09.409739Z","shell.execute_reply.started":"2023-07-07T15:39:09.248949Z","shell.execute_reply":"2023-07-07T15:39:09.408748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]#, tf.lite.OpsSet.SELECT_TF_OPS]\ntflite_model = keras_model_converter.convert()\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \ninfargs = {\"selected_columns\" : SEL_COLS}\n\nwith open('inference_args.json', \"w\") as json_file:\n    json.dump(infargs, json_file)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T15:39:15.009296Z","iopub.execute_input":"2023-07-07T15:39:15.009671Z","iopub.status.idle":"2023-07-07T15:40:11.822135Z","shell.execute_reply.started":"2023-07-07T15:39:15.009641Z","shell.execute_reply":"2023-07-07T15:40:11.821066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip  './model.tflite' './inference_args.json'","metadata":{"execution":{"iopub.status.busy":"2023-07-07T15:40:11.823986Z","iopub.execute_input":"2023-07-07T15:40:11.824353Z","iopub.status.idle":"2023-07-07T15:40:14.228334Z","shell.execute_reply.started":"2023-07-07T15:40:11.824319Z","shell.execute_reply":"2023-07-07T15:40:14.227124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tf.lite.Interpreter(\"model.tflite\")\n\nREQUIRED_SIGNATURE = \"serving_default\"\nREQUIRED_OUTPUT = \"outputs\"\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\nif REQUIRED_SIGNATURE not in found_signatures:\n    raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=batch[0][0])\nprediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\nprint(prediction_str)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T15:40:14.230543Z","iopub.execute_input":"2023-07-07T15:40:14.230931Z","iopub.status.idle":"2023-07-07T15:40:14.632617Z","shell.execute_reply.started":"2023-07-07T15:40:14.230875Z","shell.execute_reply":"2023-07-07T15:40:14.631559Z"},"trusted":true},"execution_count":null,"outputs":[]}]}