{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This work is based on Rohith Ingilela's [notebook](https://www.kaggle.com/code/irohith/aslfr-transformer). If my work is helpful to you, remember to upvote his notebook too.\nThe code is essentially the same. I made some changes and fixed some problems:\n\n1. I used two transformer layer in the encoder instead of four.\n2. I used four attention heads instead of two.\n3. I used new tokens for SOS, EOS, and padding (very minor since Rohith used rare tokens for these purposes, but still- more 'correct').\n2. I fixed a bug (probably?) in the decoder's dropout layers, which did not have the training flag, resulting in dropout during inference. This change gave a nice bump in the score.\n3. I made the passing of the training flag explicit. I know it can be implicit since it is a kwarg, but explicit passing makes the whole thing more straightforward and maybe fix another one or two training-flag-related bugs along the way.\n4. I changed the positional encoding in the decoder from tf.keras.layers.Embedding to proper positional embeddings (i.e., the usual sines and cosines usually used for this purpose). This had a significant impact.\n5. I added positional embedding to the encoder. This, too, had a significant impact.\n\nThe rest should be very similar to Rohith's notebook.\n\n\nI hope this notebook will be a valuable stepping stone for better models. Have fun :)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom skimage.transform import resize\nfrom sklearn.model_selection import train_test_split\nimport json\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport Levenshtein as lev\nimport os\nimport gc","metadata":{"execution":{"iopub.status.busy":"2023-07-05T15:20:24.013898Z","iopub.execute_input":"2023-07-05T15:20:24.014347Z","iopub.status.idle":"2023-07-05T15:20:31.855578Z","shell.execute_reply.started":"2023-07-05T15:20:24.014311Z","shell.execute_reply":"2023-07-05T15:20:31.854552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = np.random.randint(50000)\nprint(seed)\nseed = 35567\ntf.random.set_seed(seed)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T15:20:33.80675Z","iopub.execute_input":"2023-07-05T15:20:33.807472Z","iopub.status.idle":"2023-07-05T15:20:33.814243Z","shell.execute_reply.started":"2023-07-05T15:20:33.807438Z","shell.execute_reply":"2023-07-05T15:20:33.812814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data loading","metadata":{}},{"cell_type":"markdown","source":"#### Here I use new tokens for padding, start and end of sentences. (Capitals are good since the original phrases have only lower case letters, besides numbers and various signs).","metadata":{}},{"cell_type":"code","source":"pad_token = 'P'\nstart_token = 'S'\nend_token = 'E'\npad_token_idx = 59\nstart_token_idx = 60\nend_token_idx = 61","metadata":{"execution":{"iopub.status.busy":"2023-07-05T15:20:36.188411Z","iopub.execute_input":"2023-07-05T15:20:36.188768Z","iopub.status.idle":"2023-07-05T15:20:36.193384Z","shell.execute_reply.started":"2023-07-05T15:20:36.188739Z","shell.execute_reply":"2023-07-05T15:20:36.192473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\nchar_to_num[pad_token] = pad_token_idx\nchar_to_num[start_token] = start_token_idx\nchar_to_num[end_token] = end_token_idx\nnum_to_char = {j:i for i,j in char_to_num.items()}\n\ninpdir = \"/kaggle/input/asl-fingerspelling\"\ndf = pd.read_csv(f'{inpdir}/train.csv')\n\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nRHAND_LBLS = [f'x_right_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)] + [f'z_right_hand_{i}' for i in range(21)]\nLHAND_LBLS = [ f'x_left_hand_{i}' for i in range(21)] + [ f'y_left_hand_{i}' for i in range(21)] + [ f'z_left_hand_{i}' for i in range(21)]\nPOSE_LBLS = [f'x_pose_{i}' for i in POSE] + [f'y_pose_{i}' for i in POSE] + [f'z_pose_{i}' for i in POSE]\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE]\n\nSEL_COLS = X + Y + Z\nFRAME_LEN = 128\n\nX_IDX = [i for i, col in enumerate(SEL_COLS)  if \"x_\" in col]\nY_IDX = [i for i, col in enumerate(SEL_COLS)  if \"y_\" in col]\nZ_IDX = [i for i, col in enumerate(SEL_COLS)  if \"z_\" in col]\n\nRHAND_IDX = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col]\nLHAND_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col]\nRPOSE_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE]\nLPOSE_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE]","metadata":{"execution":{"iopub.status.busy":"2023-07-05T15:20:39.140918Z","iopub.execute_input":"2023-07-05T15:20:39.141292Z","iopub.status.idle":"2023-07-05T15:20:39.293999Z","shell.execute_reply.started":"2023-07-05T15:20:39.141262Z","shell.execute_reply":"2023-07-05T15:20:39.29294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"first = pd.read_parquet(f'{inpdir}/train_landmarks/5414471.parquet')\nfirst.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:29:15.622282Z","iopub.execute_input":"2023-07-05T16:29:15.622932Z","iopub.status.idle":"2023-07-05T16:29:29.034126Z","shell.execute_reply.started":"2023-07-05T16:29:15.622901Z","shell.execute_reply":"2023-07-05T16:29:29.033043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"first[first.index == 1816825349].x_face_0.hist()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T17:13:20.327071Z","iopub.execute_input":"2023-07-05T17:13:20.327442Z","iopub.status.idle":"2023-07-05T17:13:20.694279Z","shell.execute_reply.started":"2023-07-05T17:13:20.327415Z","shell.execute_reply":"2023-07-05T17:13:20.69333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\npattern =re.compile('[^a-zA-Z]')\nset([pattern.sub('',col) for col in first.columns])\n","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:51:31.108544Z","iopub.execute_input":"2023-07-05T16:51:31.108911Z","iopub.status.idle":"2023-07-05T16:51:31.120952Z","shell.execute_reply.started":"2023-07-05T16:51:31.108882Z","shell.execute_reply":"2023-07-05T16:51:31.119888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[col for col in first.columns if 'x_right_hand_' in col]\n","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:41:32.379949Z","iopub.execute_input":"2023-07-05T16:41:32.381061Z","iopub.status.idle":"2023-07-05T16:41:32.3898Z","shell.execute_reply.started":"2023-07-05T16:41:32.381025Z","shell.execute_reply":"2023-07-05T16:41:32.388591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_pad(x):\n    if tf.shape(x)[0] < FRAME_LEN:\n        x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]))\n    else:\n        x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n    return x\n\ndef pre_process(x):\n    rhand = tf.gather(x, RHAND_IDX, axis=1)\n    lhand = tf.gather(x, LHAND_IDX, axis=1)\n    rpose = tf.gather(x, RPOSE_IDX, axis=1)\n    lpose = tf.gather(x, LPOSE_IDX, axis=1)\n    \n    rnan_idx = tf.reduce_any(tf.math.is_nan(rhand), axis=1)\n    lnan_idx = tf.reduce_any(tf.math.is_nan(lhand), axis=1)\n    \n    rnans = tf.math.count_nonzero(rnan_idx)\n    lnans = tf.math.count_nonzero(lnan_idx)\n    \n    # For dominant hand\n    if rnans > lnans:\n        hand = lhand\n        pose = lpose\n        \n        hand_x = hand[:, 0*(len(LHAND_IDX)//3) : 1*(len(LHAND_IDX)//3)]\n        hand_y = hand[:, 1*(len(LHAND_IDX)//3) : 2*(len(LHAND_IDX)//3)]\n        hand_z = hand[:, 2*(len(LHAND_IDX)//3) : 3*(len(LHAND_IDX)//3)]\n        hand = tf.concat([1-hand_x, hand_y, hand_z], axis=1)\n        \n        pose_x = pose[:, 0*(len(LPOSE_IDX)//3) : 1*(len(LPOSE_IDX)//3)]\n        pose_y = pose[:, 1*(len(LPOSE_IDX)//3) : 2*(len(LPOSE_IDX)//3)]\n        pose_z = pose[:, 2*(len(LPOSE_IDX)//3) : 3*(len(LPOSE_IDX)//3)]\n        pose = tf.concat([1-pose_x, pose_y, pose_z], axis=1)\n    else:\n        hand = rhand\n        pose = rpose\n    \n    hand_x = hand[:, 0*(len(LHAND_IDX)//3) : 1*(len(LHAND_IDX)//3)]\n    hand_y = hand[:, 1*(len(LHAND_IDX)//3) : 2*(len(LHAND_IDX)//3)]\n    hand_z = hand[:, 2*(len(LHAND_IDX)//3) : 3*(len(LHAND_IDX)//3)]\n    hand = tf.concat([hand_x[..., tf.newaxis], hand_y[..., tf.newaxis], hand_z[..., tf.newaxis]], axis=-1)\n    \n    mean = tf.math.reduce_mean(hand, axis=1)[:, tf.newaxis, :]\n    std = tf.math.reduce_std(hand, axis=1)[:, tf.newaxis, :]\n    hand = (hand - mean) / std\n\n    pose_x = pose[:, 0*(len(LPOSE_IDX)//3) : 1*(len(LPOSE_IDX)//3)]\n    pose_y = pose[:, 1*(len(LPOSE_IDX)//3) : 2*(len(LPOSE_IDX)//3)]\n    pose_z = pose[:, 2*(len(LPOSE_IDX)//3) : 3*(len(LPOSE_IDX)//3)]\n    pose = tf.concat([pose_x[..., tf.newaxis], pose_y[..., tf.newaxis], pose_z[..., tf.newaxis]], axis=-1)\n    \n    x = tf.concat([hand, pose], axis=1)\n    x = resize_pad(x)\n    \n    x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n    x = tf.reshape(x, (FRAME_LEN, len(LHAND_IDX) + len(LPOSE_IDX)))\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:22.0566Z","iopub.execute_input":"2023-06-15T19:35:22.05692Z","iopub.status.idle":"2023-06-15T19:35:22.076744Z","shell.execute_reply.started":"2023-06-15T19:35:22.056892Z","shell.execute_reply":"2023-06-15T19:35:22.075517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table = tf.lookup.StaticHashTable(\n    initializer=tf.lookup.KeyValueTensorInitializer(\n        keys=list(char_to_num.keys()),\n        values=list(char_to_num.values()),\n    ),\n    default_value=tf.constant(-1),\n    name=\"class_weight\"\n)\n\ndef preprocess_fn(landmarks, phrase):\n    phrase = start_token + phrase + end_token\n    phrase = tf.strings.bytes_split(phrase)\n    phrase = table.lookup(phrase)\n    phrase = tf.pad(phrase, paddings=[[0, 64 - tf.shape(phrase)[0]]], mode = 'CONSTANT',\n                    constant_values = pad_token_idx)\n    return pre_process(landmarks), phrase\n\ndef decode_fn(record_bytes):\n    schema = {COL: tf.io.VarLenFeature(dtype=tf.float32) for COL in SEL_COLS}\n    schema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n    features = tf.io.parse_single_example(record_bytes, schema)\n    phrase = features[\"phrase\"]\n    landmarks = ([tf.sparse.to_dense(features[COL]) for COL in SEL_COLS])\n    landmarks = tf.transpose(landmarks)\n    \n    return landmarks, phrase\n\ninpdir = \"/kaggle/input/aslfr-parquets-to-tfrecords-cleaned\"\ntffiles = df.file_id.map(lambda x: f'{inpdir}/tfds/{x}.tfrecord').unique()\n\nbatch_size = 32\nval_len = int(0.05 * len(tffiles))\n\ntrain_dataset = tf.data.TFRecordDataset(tffiles[val_len:]).map(decode_fn).map(preprocess_fn).shuffle(30000, reshuffle_each_iteration=True).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)\nval_dataset = tf.data.TFRecordDataset(tffiles[:val_len]).map(decode_fn).map(preprocess_fn).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:22.078356Z","iopub.execute_input":"2023-06-15T19:35:22.078759Z","iopub.status.idle":"2023-06-15T19:35:26.757021Z","shell.execute_reply.started":"2023-06-15T19:35:22.078725Z","shell.execute_reply":"2023-06-15T19:35:26.755973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The model","metadata":{}},{"cell_type":"markdown","source":"#### Here I implemented proper positional embeddings for both the encoder and the decoder.","metadata":{}},{"cell_type":"code","source":"class TokenEmbedding(layers.Layer):\n    def __init__(self, num_vocab=1000, maxlen=100, num_hid=64):\n        super().__init__()\n        self.num_hid = num_hid\n        self.emb = tf.keras.layers.Embedding(num_vocab, num_hid)\n        #self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=num_hid)\n        '''\n        self.pos_emb = tf.math.divide(\n            self.positional_encoding(maxlen-1, num_hid),\n            tf.math.sqrt(tf.cast(num_hid, tf.float32)))\n        '''\n        self.pos_emb = self.positional_encoding(maxlen-1, num_hid)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        x = self.emb(x)\n        x = tf.math.multiply(x, tf.math.sqrt(tf.cast(self.num_hid, tf.float32)))\n        '''\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        return x + positions\n        '''\n        return x + self.pos_emb[:maxlen, :]\n    \n    def positional_encoding(self, maxlen, num_hid):\n        depth = num_hid/2\n        positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n        depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n        angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n        angle_rads = tf.linalg.matmul(positions, angle_rates)\n        pos_encoding = tf.concat(\n          [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n          axis=-1) \n        return pos_encoding\n\n\nclass LandmarkEmbedding(layers.Layer):\n    def __init__(self, num_hid=64, maxlen=100):\n        super().__init__()\n        self.conv1 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n        self.conv2 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n        self.conv3 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n        self.pos_emb = self.positional_encoding(maxlen, num_hid)\n        self.maxlen = maxlen\n        self.num_hid = num_hid\n\n    def call(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        \n        x = tf.math.multiply(x, tf.math.sqrt(tf.cast(self.num_hid, tf.float32)))\n        x = x + self.pos_emb\n        \n        return x\n    \n    def positional_encoding(self, maxlen, num_hid):\n        depth = num_hid/2\n        positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n        depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n        angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n        angle_rads = tf.linalg.matmul(positions, angle_rates)\n        pos_encoding = tf.concat(\n          [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n          axis=-1) \n        return pos_encoding","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:26.759065Z","iopub.execute_input":"2023-06-15T19:35:26.759796Z","iopub.status.idle":"2023-06-15T19:35:26.777322Z","shell.execute_reply.started":"2023-06-15T19:35:26.759761Z","shell.execute_reply":"2023-06-15T19:35:26.776362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerEncoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, rate=0.1):\n        super().__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dense(embed_dim),\n            ]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:26.77873Z","iopub.execute_input":"2023-06-15T19:35:26.779918Z","iopub.status.idle":"2023-06-15T19:35:26.794843Z","shell.execute_reply.started":"2023-06-15T19:35:26.779882Z","shell.execute_reply":"2023-06-15T19:35:26.793844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Here I added the training flag to the TransformerDecoder's Dropout layers.","metadata":{}},{"cell_type":"code","source":"class TransformerDecoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, dropout_rate=0.1):\n        super().__init__()\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = layers.LayerNormalization(epsilon=1e-6)\n        self.self_att = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=embed_dim\n        )\n        self.enc_att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.self_dropout = layers.Dropout(0.5)\n        self.enc_dropout = layers.Dropout(0.1)\n        self.ffn_dropout = layers.Dropout(0.1)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dense(embed_dim),\n            ]\n        )\n\n    def causal_attention_mask(self, batch_size, n_dest, n_src, dtype):\n        \"\"\"Masks the upper half of the dot product matrix in self attention.\n\n        This prevents flow of information from future tokens to current token.\n        1's in the lower triangle, counting from the lower right corner.\n        \"\"\"\n        i = tf.range(n_dest)[:, None]\n        j = tf.range(n_src)\n        m = i >= j - n_src + n_dest\n        mask = tf.cast(m, dtype)\n        mask = tf.reshape(mask, [1, n_dest, n_src])\n        mult = tf.concat(\n            [batch_size[..., tf.newaxis], tf.constant([1, 1], dtype=tf.int32)], 0\n        )\n        return tf.tile(mask, mult)\n\n    def call(self, enc_out, target, training):\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        seq_len = input_shape[1]\n        causal_mask = self.causal_attention_mask(batch_size, seq_len, seq_len, tf.bool)\n        target_att = self.self_att(target, target, attention_mask=causal_mask)\n        target_norm = self.layernorm1(target + self.self_dropout(target_att, training = training))\n        enc_out = self.enc_att(target_norm, enc_out)\n        enc_out_norm = self.layernorm2(self.enc_dropout(enc_out, training = training) + target_norm)\n        ffn_out = self.ffn(enc_out_norm)\n        ffn_out_norm = self.layernorm3(enc_out_norm + self.ffn_dropout(ffn_out, training = training))\n        return ffn_out_norm","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:26.79656Z","iopub.execute_input":"2023-06-15T19:35:26.796931Z","iopub.status.idle":"2023-06-15T19:35:26.812332Z","shell.execute_reply.started":"2023-06-15T19:35:26.796897Z","shell.execute_reply":"2023-06-15T19:35:26.811351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Here I made the passing of the training flag explicit.","metadata":{}},{"cell_type":"code","source":"class Transformer(keras.Model):\n    def __init__(\n        self,\n        num_hid=64,\n        num_head=2,\n        num_feed_forward=128,\n        source_maxlen=100,\n        target_maxlen=100,\n        num_layers_enc=4,\n        num_layers_dec=1,\n        num_classes=60,\n    ):\n        super().__init__()\n        self.loss_metric = keras.metrics.Mean(name=\"loss\")\n        self.num_layers_enc = num_layers_enc\n        self.num_layers_dec = num_layers_dec\n        self.target_maxlen = target_maxlen\n        self.num_classes = num_classes\n\n        self.enc_input = LandmarkEmbedding(num_hid=num_hid, maxlen=source_maxlen)\n        self.dec_input = TokenEmbedding(\n            num_vocab=num_classes, maxlen=target_maxlen, num_hid=num_hid\n        )\n\n        self.encoder = keras.Sequential(\n            [self.enc_input]\n            + [\n                TransformerEncoder(num_hid, num_head, num_feed_forward)\n                for _ in range(num_layers_enc)\n            ]\n        )\n\n        for i in range(num_layers_dec):\n            setattr(\n                self,\n                f\"dec_layer_{i}\",\n                TransformerDecoder(num_hid, num_head, num_feed_forward),\n            )\n\n        self.classifier = layers.Dense(num_classes)\n\n    def decode(self, enc_out, target, training):\n        y = self.dec_input(target)\n        for i in range(self.num_layers_dec):\n            y = getattr(self, f\"dec_layer_{i}\")(enc_out, y, training)\n        return y\n\n    def call(self, inputs, training):\n        source = inputs[0]\n        target = inputs[1]\n        x = self.encoder(source, training)\n        y = self.decode(x, target, training)\n        return self.classifier(y)\n\n    @property\n    def metrics(self):\n        return [self.loss_metric]\n\n    def train_step(self, batch):\n        \"\"\"Processes one batch inside model.fit().\"\"\"\n        source = batch[0]\n        target = batch[1]\n\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        \n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        with tf.GradientTape() as tape:\n            preds = self([source, dec_input])\n            one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n            mask = tf.math.logical_not(tf.math.equal(dec_target, pad_token_idx))\n            loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        trainable_vars = self.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n        self.loss_metric.update_state(loss)\n        return {\"loss\": self.loss_metric.result()}\n\n    def test_step(self, batch):        \n        source = batch[0]\n        target = batch[1]\n\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        \n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        preds = self([source, dec_input])\n        one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n        mask = tf.math.logical_not(tf.math.equal(dec_target, pad_token_idx))\n        loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        self.loss_metric.update_state(loss)\n        return {\"loss\": self.loss_metric.result()}\n\n    def generate(self, source, target_start_token_idx):\n        \"\"\"Performs inference over one batch of inputs using greedy decoding.\"\"\"\n        bs = tf.shape(source)[0]\n        enc = self.encoder(source, training = False)\n        dec_input = tf.ones((bs, 1), dtype=tf.int32) * target_start_token_idx\n        dec_logits = []\n        for i in range(self.target_maxlen - 1):\n            dec_out = self.decode(enc, dec_input, training = False)\n            logits = self.classifier(dec_out)\n            logits = tf.argmax(logits, axis=-1, output_type=tf.int32)\n            last_logit = logits[:, -1][..., tf.newaxis]\n            dec_logits.append(last_logit)\n            dec_input = tf.concat([dec_input, last_logit], axis=-1)\n        return dec_input","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:26.817523Z","iopub.execute_input":"2023-06-15T19:35:26.817852Z","iopub.status.idle":"2023-06-15T19:35:26.839986Z","shell.execute_reply.started":"2023-06-15T19:35:26.817827Z","shell.execute_reply":"2023-06-15T19:35:26.838985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch = next(iter(val_dataset))\nidx_to_char = list(char_to_num.keys())\n\nmodel = Transformer(\n    num_hid=200,\n    num_head=4,\n    num_feed_forward=400,\n    source_maxlen = FRAME_LEN,\n    target_maxlen=64,\n    num_layers_enc=2,\n    num_layers_dec=1,\n    num_classes=62,\n)\n\n\nloss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.1)\n\n\noptimizer = keras.optimizers.Adam(0.0001)\nmodel.compile(optimizer=optimizer, loss=loss_fn)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:26.843577Z","iopub.execute_input":"2023-06-15T19:35:26.843877Z","iopub.status.idle":"2023-06-15T19:35:29.23997Z","shell.execute_reply.started":"2023-06-15T19:35:26.843852Z","shell.execute_reply":"2023-06-15T19:35:29.239008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nhistory = model.fit(train_dataset, verbose = 2, validation_data=val_dataset, epochs=13)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:29.978015Z","iopub.execute_input":"2023-06-15T19:35:29.978327Z","iopub.status.idle":"2023-06-15T19:35:55.712579Z","shell.execute_reply.started":"2023-06-15T19:35:29.978301Z","shell.execute_reply":"2023-06-15T19:35:55.71159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model.summary())","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:55.713865Z","iopub.execute_input":"2023-06-15T19:35:55.714226Z","iopub.status.idle":"2023-06-15T19:35:55.748943Z","shell.execute_reply.started":"2023-06-15T19:35:55.714189Z","shell.execute_reply":"2023-06-15T19:35:55.748023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:55.75014Z","iopub.execute_input":"2023-06-15T19:35:55.750464Z","iopub.status.idle":"2023-06-15T19:35:56.05217Z","shell.execute_reply.started":"2023-06-15T19:35:55.750431Z","shell.execute_reply":"2023-06-15T19:35:56.0511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation","metadata":{}},{"cell_type":"code","source":"batches = [batch for batch in val_dataset]\n\npreds_list = []\nground_truth_list = []\n\nfor batch in batches[:1]:\n    source = batch[0]\n    target = batch[1].numpy()\n    bs = tf.shape(source)[0]\n    preds = model.generate(source, start_token_idx)\n    preds = preds.numpy()\n\n    for i in range(bs):\n        target_text = \"\".join([idx_to_char[_] for _ in target[i, :]])\n        ground_truth_list.append(target_text.replace('P', ''))\n        prediction = \"\"\n        for idx in preds[i, :]:\n            prediction += idx_to_char[idx]\n            if idx == end_token_idx:\n                break\n        preds_list.append(prediction)\n\nfor i in range(10):\n    print(ground_truth_list[i])\n    print(preds_list[i])\n    print('\\n~~~\\n')","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:56.053651Z","iopub.execute_input":"2023-06-15T19:35:56.054113Z","iopub.status.idle":"2023-06-15T19:35:58.051585Z","shell.execute_reply.started":"2023-06-15T19:35:56.054077Z","shell.execute_reply":"2023-06-15T19:35:58.050572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ground_truth_processed = [ground_truth_list[i][1:-1] for i in range(len(ground_truth_list))]\npreds_list_processed = [preds_list[i][1:-1] for i in range(len(preds_list))]\nlev_dist = [lev.distance(ground_truth_processed[i], preds_list_processed[i]) \n            for i in range(len(preds_list_processed))]\nN = [len(phrase) for phrase in ground_truth_processed]\n\nprint('Validation score: '+str((np.sum(N) - np.sum(lev_dist))/np.sum(N)))","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:39:58.869802Z","iopub.execute_input":"2023-06-15T19:39:58.870162Z","iopub.status.idle":"2023-06-15T19:39:58.878325Z","shell.execute_reply.started":"2023-06-15T19:39:58.870132Z","shell.execute_reply":"2023-06-15T19:39:58.877164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TFLiteModel","metadata":{}},{"cell_type":"code","source":" class TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n        self.target_start_token_idx = start_token_idx\n        self.target_end_token_idx = end_token_idx\n        # Load the feature generation and main models\n        self.model = model\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(SEL_COLS)], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs, training=False):\n        # Preprocess Data\n        x = tf.cast(inputs, tf.float32)\n        x = x[None]\n        x = tf.cond(tf.shape(x)[1] == 0, lambda: tf.zeros((1, 1, len(SEL_COLS))), lambda: tf.identity(x))\n        x = x[0]\n        x = pre_process(x)\n        x = x[None]\n        x = self.model.generate(x, self.target_start_token_idx)\n        x = x[0]\n        idx = tf.argmax(tf.cast(tf.equal(x, self.target_end_token_idx), tf.int32))\n        idx = tf.where(tf.math.less(idx, 1), tf.constant(2, dtype=tf.int64), idx)\n        x = x[1:idx]\n        x = tf.one_hot(x, 59)\n        return {'outputs': x}\n    \ntflitemodel_base = TFLiteModel(model)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:58.306108Z","iopub.status.idle":"2023-06-15T19:35:58.307025Z","shell.execute_reply.started":"2023-06-15T19:35:58.306792Z","shell.execute_reply":"2023-06-15T19:35:58.306813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights(\"model.h5\")","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:58.308228Z","iopub.status.idle":"2023-06-15T19:35:58.309231Z","shell.execute_reply.started":"2023-06-15T19:35:58.309002Z","shell.execute_reply":"2023-06-15T19:35:58.309024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]#, tf.lite.OpsSet.SELECT_TF_OPS]\ntflite_model = keras_model_converter.convert()\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \ninfargs = {\"selected_columns\" : SEL_COLS}\n\nwith open('inference_args.json', \"w\") as json_file:\n    json.dump(infargs, json_file)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:58.310282Z","iopub.status.idle":"2023-06-15T19:35:58.311346Z","shell.execute_reply.started":"2023-06-15T19:35:58.311113Z","shell.execute_reply":"2023-06-15T19:35:58.311135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip  './model.tflite' './inference_args.json'","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:58.31263Z","iopub.status.idle":"2023-06-15T19:35:58.313098Z","shell.execute_reply.started":"2023-06-15T19:35:58.312868Z","shell.execute_reply":"2023-06-15T19:35:58.312889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tf.lite.Interpreter(\"model.tflite\")\n\nREQUIRED_SIGNATURE = \"serving_default\"\nREQUIRED_OUTPUT = \"outputs\"\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\nif REQUIRED_SIGNATURE not in found_signatures:\n    raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=batch[0][0])\nprediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\nprint(prediction_str)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:35:58.31445Z","iopub.status.idle":"2023-06-15T19:35:58.316141Z","shell.execute_reply.started":"2023-06-15T19:35:58.315905Z","shell.execute_reply":"2023-06-15T19:35:58.315927Z"},"trusted":true},"execution_count":null,"outputs":[]}]}