{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport json\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-17T03:06:37.214913Z","iopub.execute_input":"2023-08-17T03:06:37.215179Z","iopub.status.idle":"2023-08-17T03:06:45.790437Z","shell.execute_reply.started":"2023-08-17T03:06:37.215153Z","shell.execute_reply":"2023-08-17T03:06:45.789367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\nnum_to_char = {j:i for i,j in char_to_num.items()}\n\ninpdir = \"/kaggle/input/asl-fingerspelling\"\ndf = pd.read_csv(f'{inpdir}/train.csv')\n\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE]\n\nFEATURE_COLUMNS = X + Y\nFRAME_LEN = 128\n\nRHAND_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"right\" in col]\nLHAND_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if  \"left\" in col]\n\nRHAND_IDX_X = [i for i, col in enumerate(FEATURE_COLUMNS) if \"right\" in col and \"x\" in col]\nLHAND_IDX_X = [i for i, col in enumerate(FEATURE_COLUMNS) if \"left\" in col and \"x\" in col]\nRPOSE_IDX_X = [i for i, col in enumerate(FEATURE_COLUMNS) if \"pose\" in col and int(col[-2:]) in RPOSE and \"x\" in col]\nLPOSE_IDX_X = [i for i, col in enumerate(FEATURE_COLUMNS) if \"pose\" in col and int(col[-2:]) in LPOSE and \"x\" in col]\n\nRHAND_IDX_Y = [i for i, col in enumerate(FEATURE_COLUMNS) if \"right\" in col and \"y\" in col]\nLHAND_IDX_Y = [i for i, col in enumerate(FEATURE_COLUMNS) if \"left\" in col and \"y\" in col]\nRPOSE_IDX_Y = [i for i, col in enumerate(FEATURE_COLUMNS) if \"pose\" in col and int(col[-2:]) in RPOSE and \"y\" in col]\nLPOSE_IDX_Y = [i for i, col in enumerate(FEATURE_COLUMNS) if \"pose\" in col and int(col[-2:]) in LPOSE and \"y\" in col]","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:45.792296Z","iopub.execute_input":"2023-08-17T03:06:45.793063Z","iopub.status.idle":"2023-08-17T03:06:45.958969Z","shell.execute_reply.started":"2023-08-17T03:06:45.793023Z","shell.execute_reply":"2023-08-17T03:06:45.957916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_pad(x):\n    if tf.shape(x)[0] < FRAME_LEN:\n        x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]), constant_values=float(\"NaN\"))\n    else:\n        x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:45.960378Z","iopub.execute_input":"2023-08-17T03:06:45.960746Z","iopub.status.idle":"2023-08-17T03:06:45.969947Z","shell.execute_reply.started":"2023-08-17T03:06:45.960709Z","shell.execute_reply":"2023-08-17T03:06:45.968811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pre_process(x):\n    rhand_x = tf.gather(x, RHAND_IDX_X, axis=1)\n    rhand_y = tf.gather(x, RHAND_IDX_Y, axis=1)\n\n    lhand_x = tf.gather(x, LHAND_IDX_X, axis=1)\n    lhand_y = tf.gather(x, LHAND_IDX_Y, axis=1)\n\n    rpose_x = tf.gather(x, RPOSE_IDX_X, axis=1)\n    rpose_y = tf.gather(x, RPOSE_IDX_Y, axis=1)\n    \n    lpose_x = tf.gather(x, LPOSE_IDX_X, axis=1)\n    lpose_y = tf.gather(x, LPOSE_IDX_Y, axis=1)\n        \n    rhand = tf.concat([rhand_x[..., tf.newaxis], rhand_y[..., tf.newaxis]], axis=-1)\n    lhand = tf.concat([lhand_x[..., tf.newaxis], lhand_y[..., tf.newaxis]], axis=-1)\n    rpose = tf.concat([rpose_x[..., tf.newaxis], rpose_y[..., tf.newaxis]], axis=-1)\n    lpose = tf.concat([lpose_x[..., tf.newaxis], lpose_y[..., tf.newaxis]], axis=-1)\n    \n    hand =  tf.concat([rhand, lhand], axis=1)\n    hand = tf.where(tf.math.is_nan(hand), 0.0, hand)\n    mask = tf.math.not_equal(tf.reduce_sum(hand, axis=[1, 2]), 0.0)\n        \n    rhand = rhand[mask]\n    lhand = lhand[mask]\n    rpose = rpose[mask]\n    lpose = lpose[mask]\n    \n    rHandMean = tf.math.reduce_mean(rhand, axis=1)[:, tf.newaxis, :]\n    rHandStd = tf.math.reduce_std(rhand, axis=1)[:, tf.newaxis, :]\n    rhand = (rhand - rHandMean) / rHandStd\n    \n    lHandMean = tf.math.reduce_mean(lhand, axis=1)[:, tf.newaxis, :]\n    lHandStd = tf.math.reduce_std(lhand, axis=1)[:, tf.newaxis, :]\n    lhand = (lhand - lHandMean) / lHandStd\n    \n    rPoseMean = tf.math.reduce_mean(rpose, axis=1)[:, tf.newaxis, :]\n    rPoseStd = tf.math.reduce_std(rpose, axis=1)[:, tf.newaxis, :]\n    rpose = (rpose - rPoseMean) / rPoseStd\n    \n    lPoseMean = tf.math.reduce_mean(lpose, axis=1)[:, tf.newaxis, :]\n    lPoseStd = tf.math.reduce_std(lpose, axis=1)[:, tf.newaxis, :]\n    lpose = (lpose - lPoseMean) / lPoseStd\n\n    x = tf.concat([rhand, lhand, rpose, lpose], axis=1)\n    x = resize_pad(x)\n    s = tf.shape(x)\n    \n    x = tf.reshape(x, (s[0], s[1]*s[2]))\n    x = tf.where(tf.math.is_nan(x), 0.0, x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:45.973787Z","iopub.execute_input":"2023-08-17T03:06:45.97408Z","iopub.status.idle":"2023-08-17T03:06:45.991093Z","shell.execute_reply.started":"2023-08-17T03:06:45.974054Z","shell.execute_reply":"2023-08-17T03:06:45.989983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=FEATURE_COLUMNS)\n\nfile_id = df.file_id.iloc[0]\ninpdir = \"/kaggle/input/asl-fingerspelling/train_landmarks\"\npqfile = f\"{inpdir}/{file_id}.parquet\"\nseq_refs = df.loc[df.file_id == file_id]\nseqs = load_relevant_data_subset(pqfile)\n\nseq_id = seq_refs.sequence_id.iloc[0]\nframes = seqs.iloc[seqs.index == seq_id]\nphrase = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])\n\npre = pre_process(frames)\nINPUT_SHAPE = list(pre.shape)\nprint(INPUT_SHAPE)","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:45.992753Z","iopub.execute_input":"2023-08-17T03:06:45.993136Z","iopub.status.idle":"2023-08-17T03:06:51.979834Z","shell.execute_reply.started":"2023-08-17T03:06:45.9931Z","shell.execute_reply":"2023-08-17T03:06:51.978797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table = tf.lookup.StaticHashTable(\n    initializer=tf.lookup.KeyValueTensorInitializer(\n        keys=list(char_to_num.keys()),\n        values=list(char_to_num.values()),\n    ),\n    default_value=tf.constant(-1),\n    name=\"class_weight\"\n)\n\nmask_idx = char_to_num['#']\n\ndef preprocess_fn(landmarks, phrase):\n    phrase = ';' + phrase + '['\n    phrase = tf.strings.bytes_split(phrase)\n    phrase = table.lookup(phrase)\n    phrase = tf.pad(phrase, paddings=[[0, 64 - tf.shape(phrase)[0]]], constant_values=mask_idx)\n    return pre_process(landmarks), phrase\n\ndef decode_fn(record_bytes):\n    schema = {COL: tf.io.VarLenFeature(dtype=tf.float32) for COL in FEATURE_COLUMNS}\n    schema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n    features = tf.io.parse_single_example(record_bytes, schema)\n    phrase = features[\"phrase\"]\n    landmarks = ([tf.sparse.to_dense(features[COL]) for COL in FEATURE_COLUMNS])\n    landmarks = tf.transpose(landmarks)\n    return landmarks, phrase","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:51.981059Z","iopub.execute_input":"2023-08-17T03:06:51.981432Z","iopub.status.idle":"2023-08-17T03:06:51.997911Z","shell.execute_reply.started":"2023-08-17T03:06:51.981398Z","shell.execute_reply":"2023-08-17T03:06:51.996963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import shutil\n# import pyarrow.parquet as pq\n\n# dataset_df = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n# # Set length of frames to 128\n# FRAME_LEN = 128\n\n# # Create directory to store the new data\n# if not os.path.isdir(\"preprocessed\"):\n#     os.mkdir(\"preprocessed\")\n# else:\n#     shutil.rmtree(\"preprocessed\")\n#     os.mkdir(\"preprocessed\")\n\n# # Loop through each file_id\n# for file_id in tqdm(dataset_df.file_id.unique()):\n#     # Parquet file name\n#     pq_file = f\"/kaggle/input/asl-fingerspelling/train_landmarks/{file_id}.parquet\"\n#     # Filter train.csv and fetch entries only for the relevant file_id\n#     file_df = dataset_df.loc[dataset_df[\"file_id\"] == file_id]\n#     # Fetch the parquet file\n#     parquet_df = pq.read_table(f\"/kaggle/input/asl-fingerspelling/train_landmarks/{str(file_id)}.parquet\",\n#                               columns=['sequence_id'] + FEATURE_COLUMNS).to_pandas()\n#     # File name for the updated data\n#     tf_file = f\"preprocessed/{file_id}.tfrecord\"\n#     parquet_numpy = parquet_df.to_numpy()\n#     # Initialize the pointer to write the output of \n#     # each `for loop` below as a sequence into the file.\n#     with tf.io.TFRecordWriter(tf_file) as file_writer:\n#         # Loop through each sequence in file.\n#         for seq_id, phrase in zip(file_df.sequence_id, file_df.phrase):\n#             # Fetch sequence data\n#             frames = parquet_numpy[parquet_df.index == seq_id]\n            \n#             # Calculate the number of NaN values in each hand landmark\n#             r_nonan = np.sum(np.sum(np.isnan(frames[:, RHAND_IDX]), axis = 1) == 0)\n#             l_nonan = np.sum(np.sum(np.isnan(frames[:, LHAND_IDX]), axis = 1) == 0)\n#             no_nan = max(r_nonan, l_nonan)\n            \n#             if 2*len(phrase)<no_nan:\n#                 features = {FEATURE_COLUMNS[i]: tf.train.Feature(\n#                     float_list=tf.train.FloatList(value=frames[:, i])) for i in range(len(FEATURE_COLUMNS))}\n#                 features[\"phrase\"] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[bytes(phrase, 'utf-8')]))\n#                 record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n#                 file_writer.write(record_bytes)","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:51.999388Z","iopub.execute_input":"2023-08-17T03:06:51.99985Z","iopub.status.idle":"2023-08-17T03:06:52.006988Z","shell.execute_reply.started":"2023-08-17T03:06:51.999817Z","shell.execute_reply":"2023-08-17T03:06:52.00601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inpdir = \"/kaggle/input/tfrecords/preprocessed\"\ntffiles = df.file_id.map(lambda x: f'{inpdir}/{x}.tfrecord').unique()\n\nbatch_size = 64\n#val_len = int(0.05 * len(tffiles))\nval_len = 1\n\ntrain_dataset = tf.data.TFRecordDataset(tffiles[val_len:]).map(decode_fn).map(preprocess_fn).shuffle(buffer_size=500).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)\nval_dataset = tf.data.TFRecordDataset(tffiles[:val_len]).map(decode_fn).map(preprocess_fn).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)\ntest_dataset = tf.data.TFRecordDataset(tffiles).map(decode_fn).prefetch(buffer_size=tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:52.008388Z","iopub.execute_input":"2023-08-17T03:06:52.008979Z","iopub.status.idle":"2023-08-17T03:06:54.037622Z","shell.execute_reply.started":"2023-08-17T03:06:52.008942Z","shell.execute_reply":"2023-08-17T03:06:54.036605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TokenEmbedding(layers.Layer):\n    def __init__(self, num_vocab=1000, maxlen=100, num_hid=64):\n        super().__init__()\n        self.emb = tf.keras.layers.Embedding(num_vocab, num_hid)\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=num_hid)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        x = self.emb(x)\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        return x + positions\n\n\nclass LandmarkEmbedding(layers.Layer):\n    def __init__(self, num_hid=64, maxlen=100):\n        super().__init__()\n        self.conv1 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n        self.conv2 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n        self.conv3 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n\n    def call(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:54.03911Z","iopub.execute_input":"2023-08-17T03:06:54.039462Z","iopub.status.idle":"2023-08-17T03:06:54.0535Z","shell.execute_reply.started":"2023-08-17T03:06:54.039427Z","shell.execute_reply":"2023-08-17T03:06:54.052508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerEncoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, rate=0.1):\n        super().__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dense(embed_dim),\n            ]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:54.058777Z","iopub.execute_input":"2023-08-17T03:06:54.059194Z","iopub.status.idle":"2023-08-17T03:06:54.069396Z","shell.execute_reply.started":"2023-08-17T03:06:54.059168Z","shell.execute_reply":"2023-08-17T03:06:54.068248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerDecoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, dropout_rate=0.1):\n        super().__init__()\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = layers.LayerNormalization(epsilon=1e-6)\n        self.self_att = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=embed_dim\n        )\n        self.enc_att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.self_dropout = layers.Dropout(0.5)\n        self.enc_dropout = layers.Dropout(0.1)\n        self.ffn_dropout = layers.Dropout(0.1)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dense(embed_dim),\n            ]\n        )\n\n    def causal_attention_mask(self, batch_size, n_dest, n_src, dtype):\n        \"\"\"Masks the upper half of the dot product matrix in self attention.\n\n        This prevents flow of information from future tokens to current token.\n        1's in the lower triangle, counting from the lower right corner.\n        \"\"\"\n        i = tf.range(n_dest)[:, None]\n        j = tf.range(n_src)\n        m = i >= j - n_src + n_dest\n        mask = tf.cast(m, dtype)\n        \n        #s = tf.shape(mask)\n        #mask = tf.reshape(x, (s[0], s[1]*s[2]))\n        \n        mask = tf.reshape(mask, [1, n_dest, n_src])\n        mult = tf.concat(\n            [batch_size[..., tf.newaxis], tf.constant([1, 1], dtype=tf.int32)], 0\n        )\n        return tf.tile(mask, mult)\n\n    def call(self, enc_out, target):\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        seq_len = input_shape[1]\n        causal_mask = self.causal_attention_mask(batch_size, seq_len, seq_len, tf.bool)\n        target_att = self.self_att(target, target, attention_mask=causal_mask)\n        target_norm = self.layernorm1(target + self.self_dropout(target_att))\n        enc_out = self.enc_att(target_norm, enc_out)\n        enc_out_norm = self.layernorm2(self.enc_dropout(enc_out) + target_norm)\n        ffn_out = self.ffn(enc_out_norm)\n        ffn_out_norm = self.layernorm3(enc_out_norm + self.ffn_dropout(ffn_out))\n        return ffn_out_norm","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:54.071189Z","iopub.execute_input":"2023-08-17T03:06:54.071557Z","iopub.status.idle":"2023-08-17T03:06:54.086964Z","shell.execute_reply.started":"2023-08-17T03:06:54.071521Z","shell.execute_reply":"2023-08-17T03:06:54.085715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Transformer(keras.Model):\n    def __init__(\n        self,\n        num_hid=64,\n        num_head=2,\n        num_feed_forward=128,\n        source_maxlen=128,\n        target_maxlen=100,\n        num_layers_enc=4,\n        num_layers_dec=1,\n        num_classes=60,\n    ):\n        super().__init__()\n        self.loss_metric = keras.metrics.Mean(name=\"loss\")\n        self.acc_metric = keras.metrics.Mean(name=\"edit_dist\")\n        self.num_layers_enc = num_layers_enc\n        self.num_layers_dec = num_layers_dec\n        self.target_maxlen = target_maxlen\n        self.num_classes = num_classes\n\n        self.enc_input = LandmarkEmbedding(num_hid=num_hid, maxlen=source_maxlen)\n        self.dec_input = TokenEmbedding(\n            num_vocab=num_classes, maxlen=target_maxlen, num_hid=num_hid\n        )\n\n        self.encoder = keras.Sequential([self.enc_input] + [TransformerEncoder(num_hid, num_head, num_feed_forward) for _ in range(num_layers_enc)]\n        )\n\n        for i in range(num_layers_dec):\n            setattr(\n                self,\n                f\"dec_layer_{i}\",\n                TransformerDecoder(num_hid, num_head, num_feed_forward),\n            )\n\n        self.classifier = layers.Dense(num_classes)\n\n    def decode(self, enc_out, target):\n        y = self.dec_input(target)\n        for i in range(self.num_layers_dec):\n            y = getattr(self, f\"dec_layer_{i}\")(enc_out, y)\n        return y\n\n    def call(self, inputs):\n        source = inputs[0]\n        target = inputs[1]\n        x = self.encoder(source)\n        y = self.decode(x, target)\n        return self.classifier(y)\n\n    @property\n    def metrics(self):\n        return [self.loss_metric, self.acc_metric]\n\n    def train_step(self, batch):\n        \"\"\"Processes one batch inside model.fit().\"\"\"\n        source = batch[0]\n        target = batch[1]\n        \n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        with tf.GradientTape() as tape:\n            preds = self([source, dec_input])\n            one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n            mask = tf.math.logical_not(tf.math.equal(dec_target, mask_idx))\n            loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        trainable_vars = self.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n        self.loss_metric.update_state(loss)\n        \n        edit_dist = tf.edit_distance(tf.sparse.from_dense(target), tf.sparse.from_dense(tf.cast(tf.argmax(preds, axis=1), tf.int32)))\n        edit_dist = tf.reduce_mean(edit_dist)\n        self.acc_metric.update_state(edit_dist)\n        \n        return {\"loss\": self.loss_metric.result(), \"edit_dist\": self.acc_metric.result()}\n\n    def test_step(self, batch):\n        source = batch[0]\n        target = batch[1]\n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        preds = self([source, dec_input])\n        one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n        mask = tf.math.logical_not(tf.math.equal(dec_target, mask_idx))\n        loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        self.loss_metric.update_state(loss)\n        \n        edit_dist = tf.edit_distance(tf.sparse.from_dense(target), tf.sparse.from_dense(tf.cast(tf.argmax(preds, axis=1), tf.int32)))\n        edit_dist = tf.reduce_mean(edit_dist)\n        self.acc_metric.update_state(edit_dist)\n        \n        return {\"loss\": self.loss_metric.result(), \"edit_dist\": self.acc_metric.result()}\n\n    def generate(self, source, target_start_token_idx):\n        \"\"\"Performs inference over one batch of inputs using greedy decoding.\"\"\"\n        bs = tf.shape(source)[0]\n        enc = self.encoder(source)\n        dec_input = tf.ones((bs, 1), dtype=tf.int32) * target_start_token_idx\n        dec_logits = []\n        for i in range(self.target_maxlen - 1):\n            dec_out = self.decode(enc, dec_input)\n            logits = self.classifier(dec_out)\n            logits = tf.argmax(logits, axis=-1, output_type=tf.int32)\n            last_logit = logits[:, -1][..., tf.newaxis]\n            dec_logits.append(last_logit)\n            dec_input = tf.concat([dec_input, last_logit], axis=-1)\n        return dec_input","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:54.088573Z","iopub.execute_input":"2023-08-17T03:06:54.088931Z","iopub.status.idle":"2023-08-17T03:06:54.114448Z","shell.execute_reply.started":"2023-08-17T03:06:54.088898Z","shell.execute_reply":"2023-08-17T03:06:54.113678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DisplayOutputs(keras.callbacks.Callback):\n    def __init__(\n        self, batch, idx_to_token, target_start_token_idx=27, target_end_token_idx=28\n    ):\n        \"\"\"Displays a batch of outputs after every epoch\n\n        Args:\n            batch: A test batch containing the keys \"source\" and \"target\"\n            idx_to_token: A List containing the vocabulary tokens corresponding to their indices\n            target_start_token_idx: A start token index in the target vocabulary\n            target_end_token_idx: An end token index in the target vocabulary\n        \"\"\"\n        self.batch = batch\n        self.target_start_token_idx = target_start_token_idx\n        self.target_end_token_idx = target_end_token_idx\n        self.idx_to_char = idx_to_token\n\n    def on_epoch_end(self, epoch, logs=None):\n        source = self.batch[0]\n        target = self.batch[1].numpy()\n        bs = tf.shape(source)[0]\n        preds = self.model.generate(source, self.target_start_token_idx)\n        preds = preds.numpy()\n        for i in range(bs):\n            target_text = \"\".join([self.idx_to_char[_] for _ in target[i, :]])\n            prediction = \"\"\n            for idx in preds[i, :]:\n                prediction += self.idx_to_char[idx]\n                if idx == self.target_end_token_idx:\n                    break\n            print(f\"target:     {target_text.replace('-','')}\")\n            print(f\"prediction: {prediction}\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:54.116693Z","iopub.execute_input":"2023-08-17T03:06:54.117024Z","iopub.status.idle":"2023-08-17T03:06:54.130452Z","shell.execute_reply.started":"2023-08-17T03:06:54.116992Z","shell.execute_reply":"2023-08-17T03:06:54.129381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch = next(iter(val_dataset))\n\nidx_to_char = list(char_to_num.keys())\ndisplay_cb = DisplayOutputs(\n    batch, num_to_char, target_start_token_idx=char_to_num[';'], target_end_token_idx=char_to_num['[']\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:54.131913Z","iopub.execute_input":"2023-08-17T03:06:54.132246Z","iopub.status.idle":"2023-08-17T03:06:54.670037Z","shell.execute_reply.started":"2023-08-17T03:06:54.132212Z","shell.execute_reply":"2023-08-17T03:06:54.669025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Transformer(\n    num_hid=200,\n    num_head=2,\n    num_feed_forward=400,\n    target_maxlen=64,\n    num_layers_enc=3,\n    num_layers_dec=2,\n    num_classes=62,\n)\n\nloss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.1,)\noptimizer = keras.optimizers.Adam(0.0001)\nmodel.compile(optimizer=optimizer, loss=loss_fn)","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:54.671875Z","iopub.execute_input":"2023-08-17T03:06:54.672518Z","iopub.status.idle":"2023-08-17T03:06:54.781833Z","shell.execute_reply.started":"2023-08-17T03:06:54.672483Z","shell.execute_reply":"2023-08-17T03:06:54.780895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_dataset, validation_data=val_dataset, callbacks=[display_cb], epochs=15)","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:06:54.783415Z","iopub.execute_input":"2023-08-17T03:06:54.78378Z","iopub.status.idle":"2023-08-17T03:38:40.0204Z","shell.execute_reply.started":"2023-08-17T03:06:54.783747Z","shell.execute_reply":"2023-08-17T03:38:40.019334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TFLiteModel(tf.Module):\n    def __init__(self, model, target_start_token_idx=char_to_num[';'], target_end_token_idx=char_to_num['[']):\n        super(TFLiteModel, self).__init__()\n        self.model = model\n        self.target_start_token_idx = target_start_token_idx\n        self.target_end_token_idx = target_end_token_idx\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(FEATURE_COLUMNS)], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs, training=False):\n        # Preprocess Data\n        x = tf.cast(inputs, tf.float32)\n        x = x[None]\n        x = tf.cond(tf.shape(x)[1] == 0, lambda: tf.zeros((1, 1, len(FEATURE_COLUMNS))), lambda: tf.identity(x))\n        x = x[0]\n        x = pre_process(x)\n        x = tf.reshape(x, INPUT_SHAPE)\n        x = x[None]\n        x = self.model.generate(x, self.target_start_token_idx)\n        x = x[0]\n        idx = tf.argmax(tf.cast(tf.equal(x, self.target_end_token_idx), tf.int32))\n        idx = tf.where(tf.math.less(idx, 1), tf.constant(2, dtype=tf.int64), idx)\n        x = x[1:idx]\n        x = tf.one_hot(x, 59)\n        return {'outputs': x}\n\ntflitemodel_base = TFLiteModel(model)","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:36:06.514524Z","iopub.execute_input":"2023-08-16T04:36:06.514916Z","iopub.status.idle":"2023-08-16T04:36:06.526897Z","shell.execute_reply.started":"2023-08-16T04:36:06.514878Z","shell.execute_reply":"2023-08-16T04:36:06.525703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights(\"model.h5\")","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:36:06.528476Z","iopub.execute_input":"2023-08-16T04:36:06.528825Z","iopub.status.idle":"2023-08-16T04:36:06.675724Z","shell.execute_reply.started":"2023-08-16T04:36:06.528789Z","shell.execute_reply":"2023-08-16T04:36:06.674605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]\ntflite_model = keras_model_converter.convert()\n\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \nwith open('inference_args.json', \"w\") as f:\n    json.dump({\"selected_columns\" : FEATURE_COLUMNS}, f)\n    \n!zip submission.zip  './model.tflite' './inference_args.json'","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:36:06.678356Z","iopub.execute_input":"2023-08-16T04:36:06.679002Z","iopub.status.idle":"2023-08-16T04:37:39.53651Z","shell.execute_reply.started":"2023-08-16T04:36:06.678964Z","shell.execute_reply":"2023-08-16T04:37:39.535212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tf.lite.Interpreter(\"model.tflite\")\n\nREQUIRED_SIGNATURE = \"serving_default\"\nREQUIRED_OUTPUT = \"outputs\"\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\nif REQUIRED_SIGNATURE not in found_signatures:\n    raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\n\nfor frame, target in test_dataset.skip(1000).take(10):\n    output = prediction_fn(inputs=frame)\n    prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\n    target = target.numpy().decode(\"utf-8\")\n    print(\"pred =\", prediction_str, \"; target =\", target)","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:37:39.539573Z","iopub.execute_input":"2023-08-16T04:37:39.540481Z","iopub.status.idle":"2023-08-16T04:37:44.067399Z","shell.execute_reply.started":"2023-08-16T04:37:39.540441Z","shell.execute_reply":"2023-08-16T04:37:44.066311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit -n 10\noutput = prediction_fn(inputs=frame)","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:37:44.069205Z","iopub.execute_input":"2023-08-16T04:37:44.069864Z","iopub.status.idle":"2023-08-16T04:38:02.06512Z","shell.execute_reply.started":"2023-08-16T04:37:44.069826Z","shell.execute_reply":"2023-08-16T04:38:02.063941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from Levenshtein import distance\n\nscores = []\n\nfor frame, target in tqdm(test_dataset.take(1000)):\n    output = prediction_fn(inputs=frame)\n    prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\n    target = target.numpy().decode(\"utf-8\")\n    score = (len(target) - distance(prediction_str, target)) / len(target)\n    scores.append(score)\n    \nscores = np.array(scores)\nprint(np.sum(scores) / len(scores))","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:38:02.066598Z","iopub.execute_input":"2023-08-16T04:38:02.067042Z","iopub.status.idle":"2023-08-16T04:44:37.546788Z","shell.execute_reply.started":"2023-08-16T04:38:02.067007Z","shell.execute_reply":"2023-08-16T04:44:37.545765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}