{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Aug 23:\nnot pad the input during inference\n\n===============\n\nif rm dominant hand, what are changed:\n1. CHANNELS variable 234->360\n2. Preprocess class\n\nif use greysnow conv1d, what are changed:\n1. CFG.enc_dim variable 384 -> 256\n2. LandmarkEmbedding class\n\nif penalize decoder more\n1. get_model class (mlp_dropout_ratio x 1.5) (clf_dropout_ratio x 2)\n","metadata":{}},{"cell_type":"code","source":"# Install and Import\nimport tensorflow as tf\nimport json\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:58:57.200552Z","iopub.execute_input":"2023-08-24T20:58:57.202151Z","iopub.status.idle":"2023-08-24T20:59:08.512356Z","shell.execute_reply.started":"2023-08-24T20:58:57.20208Z","shell.execute_reply":"2023-08-24T20:59:08.510827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# charater to index\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as f:\n    CHAR2IDX = json.load(f)\n\nIDX2CHAR = {idx: char for char, idx in CHAR2IDX.items()}\n\n\nN_UNIQUE_CHARACTERS0 = len(CHAR2IDX)\nN_UNIQUE_CHARACTERS = len(CHAR2IDX) + 1 + 1 + 1\n\nPAD_IDX = len(CHAR2IDX)\nSOS_IDX = len(CHAR2IDX) + 1\nEOS_IDX = len(CHAR2IDX) + 2\n\nCHAR2IDX['E'] = EOS_IDX","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:08.514985Z","iopub.execute_input":"2023-08-24T20:59:08.51599Z","iopub.status.idle":"2023-08-24T20:59:08.529677Z","shell.execute_reply.started":"2023-08-24T20:59:08.515926Z","shell.execute_reply":"2023-08-24T20:59:08.528278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modeling - configuration\nCHANNELS = 492 # 366 # 134 # 122 # 164\nPAD_FLOAT = 0. # -100.\n\n# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n\n# Activations\nGELU = tf.keras.activations.gelu","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:08.531765Z","iopub.execute_input":"2023-08-24T20:59:08.532266Z","iopub.status.idle":"2023-08-24T20:59:08.545223Z","shell.execute_reply.started":"2023-08-24T20:59:08.53223Z","shell.execute_reply":"2023-08-24T20:59:08.543979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modeling - Embeds a landmark using fully connected layers\nclass LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n\n    # 是因为 dense layer 需要知道输入 size，所以在 build 里初始化吗？\n    def build(self, input_shape):\n        # Embedding for missing landmark in frame, initizlied with zeros\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # Embedding\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.dense(x),\n            )\n\n# Creates position embedding for each frame\nclass Embedding(tf.keras.Model):\n    def __init__(self, frame_len=128, enc_dim=384):\n        super(Embedding, self).__init__(name='embedding')\n        self.supports_masking = True\n        self.frame_len = frame_len\n        self.enc_dim = enc_dim\n\n    # 这里用 build 是因为 LandmarkEmbedding 需要知道输入 size 吗？\n    def build(self, input_shape):\n        # Positional embedding for each frame index\n        self.positional_embedding = tf.Variable(\n            initial_value=tf.zeros([self.frame_len, self.enc_dim], dtype=tf.float32),\n            trainable=True,\n            name='embedding_positional_encoder',\n        )\n        # self.positional_embedding = self.positional_encoding(self.frame_len, self.enc_dim)\n        # Embedding layer for Landmarks\n        self.dominant_hand_embedding = LandmarkEmbedding(self.enc_dim, 'dominant_hand')\n\n    # # new in m10v3\n    # def positional_encoding(self, frame_len, enc_dim):\n    #     depth = enc_dim/2\n    #     positions = tf.range(frame_len, dtype = tf.float32)[..., tf.newaxis]\n    #     depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n    #     angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n    #     angle_rads = tf.linalg.matmul(positions, angle_rates)\n    #     pos_encoding = tf.concat(\n    #       [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n    #       axis=-1)\n    #     return pos_encoding\n\n    def call(self, x, training=False):\n        # Normalize\n        # x = tf.where(tf.math.equal(x, 0.0), 0.0, (x-MEANS)/STDS)\n        # Dominant Hand\n        x = self.dominant_hand_embedding(x)\n        # Add Positional Encoding\n        x = x + self.positional_embedding\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:08.548422Z","iopub.execute_input":"2023-08-24T20:59:08.548873Z","iopub.status.idle":"2023-08-24T20:59:08.575654Z","shell.execute_reply.started":"2023-08-24T20:59:08.548822Z","shell.execute_reply":"2023-08-24T20:59:08.574071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modeling - Conv1DBlock (new in M10)\nclass ECA(tf.keras.layers.Layer):\n    def __init__(self, kernel_size=5, **kwargs):\n        super().__init__(**kwargs)\n        self.supports_masking = True\n        self.kernel_size = kernel_size\n        self.conv = tf.keras.layers.Conv1D(1, kernel_size=kernel_size, strides=1, padding=\"same\", use_bias=False)\n\n    def call(self, inputs, mask=None):\n        nn = tf.keras.layers.GlobalAveragePooling1D()(inputs, mask=mask)\n        nn = tf.expand_dims(nn, -1)\n        nn = self.conv(nn)\n        nn = tf.squeeze(nn, -1)\n        nn = tf.nn.sigmoid(nn)\n        nn = nn[:,None,:]\n        return inputs * nn\n\nclass CausalDWConv1D(tf.keras.layers.Layer):\n    def __init__(self,\n        kernel_size=17, #17\n        dilation_rate=1,\n        use_bias=False,\n        depthwise_initializer='glorot_uniform',\n        name='', **kwargs):\n        super().__init__(name=name,**kwargs)\n        self.causal_pad = tf.keras.layers.ZeroPadding1D((dilation_rate*(kernel_size-1),0),name=name + '_pad')\n        self.dw_conv = tf.keras.layers.DepthwiseConv1D(\n                            kernel_size,\n                            strides=1,\n                            dilation_rate=dilation_rate,\n                            padding='valid',\n                            use_bias=use_bias,\n                            depthwise_initializer=depthwise_initializer,\n                            name=name + '_dwconv')\n        self.supports_masking = True\n\n    def call(self, inputs):\n        x = self.causal_pad(inputs)\n        x = self.dw_conv(x)\n        return x\n\n# 这里和原版对一下，看看用 build + dense 可不可以。\nclass Conv1DBlock(tf.keras.layers.Layer):\n    def __init__(self,\n        channel_size,\n        kernel_size=17, # v1:11, v3:17\n        dilation_rate=1,\n        drop_rate=0.2,\n        expand_ratio=2, # v1:2 v2:1\n        se_ratio=0.25,\n        activation='swish',\n        name='',  **kwargs):\n\n        super().__init__(name=name,**kwargs)\n        self.dense1 = tf.keras.layers.Dense(channel_size*expand_ratio, use_bias=True, activation=activation, name=name + '_expand_conv')\n        self.causal_conv = CausalDWConv1D(kernel_size, dilation_rate=dilation_rate, use_bias=False, name=name + '_dwconv')\n        self.bn = tf.keras.layers.BatchNormalization(momentum=0.95, name=name + '_bn')\n        self.eca = ECA()\n        self.dense2 = tf.keras.layers.Dense(channel_size, use_bias=True, name=name + '_project_conv')\n        self.do = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1), name=name + '_drop')\n\n    def call(self, inputs):\n        skip = inputs\n        x = self.dense1(inputs)\n        x = self.causal_conv(x)\n        x = self.bn(x)\n        x = self.eca(x)\n        x = self.dense2(x)\n        x = self.do(x)\n        x = x + skip\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:08.577954Z","iopub.execute_input":"2023-08-24T20:59:08.578836Z","iopub.status.idle":"2023-08-24T20:59:08.60495Z","shell.execute_reply.started":"2023-08-24T20:59:08.57879Z","shell.execute_reply":"2023-08-24T20:59:08.603358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modeling - multiHead Attention\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    # m5-v11 no need to pass d_out here\n    def __init__(self, d_model, n_heads, q_len, kv_len, dropout): #, d_out=None): # d_model: emb. dim. | dec_dim | 256 frame_len: 33\n        super(MultiHeadAttention,self).__init__()\n        # Number of Units in Model\n        self.d_model = d_model\n        # Number of Attention Heads\n        self.n_heads = n_heads\n        self.kv_len = kv_len\n        self.q_len = q_len                                              # 4\n        # Number of Units in Intermediate Layers\n        self.depth = d_model // 2                                               # 128\n        # Scaling Factor Of Values\n        # m5-v14 dk should be 48, not 192 add '// self.n_heads'\n        self.scale = 1.0 / tf.math.sqrt(tf.cast(self.depth, tf.float32))        # 1/sqrt(dk) | 1/sqrt(128)\n        # Learnable Projections to Depth\n        self.wq = self.fused_mha(self.depth, q_len)                             # (B,t,128)->(B,t,4,128/4)->(B,4,t,32)\n        self.wk = self.fused_mha(self.depth, kv_len)                            # (B,T,128)->(B,T,4,128/4)->(B,4,T,32)\n        self.wv = self.fused_mha(self.depth, kv_len)                            # (B,T,128)->(B,T,4,128/4)->(B,4,T,32)\n        # Output Projection\n        # m5-v11 no need to pass d_out here\n        # self.wo = tf.keras.layers.Dense(d_model if d_out is None else d_out, use_bias=False)\n        self.wo = tf.keras.layers.Dense(d_model, use_bias=False)\n        # Softmax Activation Which Supports Masking\n        self.softmax = tf.keras.layers.Softmax()\n        # Reshaping Of Multiple Attention heads to Single Value\n        self.reshape = tf.keras.Sequential([\n            # [attention heads, number of frames, d_model] → [number of frames, n_heads, d_model // n_heads]\n            tf.keras.layers.Permute([2, 1, 3]),\n            # [number of frames, attention heads, d_model] → [number of frames, d_model]\n            tf.keras.layers.Reshape([self.q_len, self.depth]),\n        ])\n        # Output Dropout\n        self.do = tf.keras.layers.Dropout(dropout)\n        self.supports_masking = True\n\n    # Single dense layer for all attention heads\n    def fused_mha(self, dim, len):\n        return tf.keras.Sequential([\n            # Single dense layer\n            tf.keras.layers.Dense(dim, use_bias=False),\n            # Reshape to [number of frames, number of attention head, depth]\n            tf.keras.layers.Reshape([len, self.n_heads, dim // self.n_heads]),\n            # Permutate to [number of attention heads, number of frames, depth]\n            tf.keras.layers.Permute([2, 1, 3]),\n        ])\n\n    def call(self, q, k, v, attention_mask=None, training=False):\n        # Projections to attention heads\n        Q = self.wq(q)                                                          # (B,t,256)->(B,t,128)->(B,t,4,128/4)->(B,4,t,32)\n        K = self.wk(k)                                                          # (B,T,256)->(B,T,128)->(B,T,4,128/4)->(B,4,T,32)\n        V = self.wv(v)                                                          # (B,T,256)->(B,T,128)->(B,T,4,128/4)->(B,4,T,32)\n        # Matrix multiply QxK to acquire attention scores\n        x = tf.matmul(Q, K, transpose_b=True) * self.scale                      # (B,4,t,32)*(B,4,32,T)->(B,4,t,T)\n        # Softmax attention scores and Multiply with Values\n        x = self.softmax(x, mask=attention_mask) @ V                            # (B,4,t,T)*(B,4,T,32)->(B,4,t,32)\n        # Reshape to flatten attention heads\n        x = self.reshape(x)                                                     # (B,4,t,32)->(B,t,4,32)->(B,t,128)\n        # Output projection\n        x = self.wo(x)\n        # Dropout\n        x = self.do(x, training=training)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:08.607052Z","iopub.execute_input":"2023-08-24T20:59:08.607434Z","iopub.status.idle":"2023-08-24T20:59:08.637164Z","shell.execute_reply.started":"2023-08-24T20:59:08.607403Z","shell.execute_reply":"2023-08-24T20:59:08.633984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modeling - encoder\nclass Encoder(tf.keras.Model):\n    def __init__(self, n_enc_blocks, enc_dim, dec_dim, n_mha_heads, mha_dropout_ratio, mlp_ratio, mlp_dropout_ratio, frame_len):\n        super(Encoder, self).__init__(name='encoder')\n        self.n_enc_blocks = n_enc_blocks\n        self.enc_dim = enc_dim\n        self.dec_dim = dec_dim\n        self.n_mha_heads = n_mha_heads # MHA\n        self.mha_dropout_ratio = mha_dropout_ratio # HMA\n        self.mlp_ratio = mlp_ratio # MLP\n        self.mlp_dropout_ratio = mlp_dropout_ratio # MLP\n        self.frame_len = frame_len\n        self.supports_masking = True # new in V6\n\n    def build(self, input_shape):\n        self.convs = []\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.n_enc_blocks):\n            self.convs.append(tf.keras.Sequential([\n                Conv1DBlock(self.enc_dim, name=f'convblk_{i}a'),\n                Conv1DBlock(self.enc_dim, name=f'convblk_{i}b'),\n                Conv1DBlock(self.enc_dim, name=f'convblk_{i}c'),\n                ]))\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # M8-v1\n            self.mhas.append(MultiHeadAttention(self.enc_dim, self.n_mha_heads, self.frame_len, self.frame_len, self.mha_dropout_ratio))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(self.enc_dim * self.mlp_ratio, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False),\n                tf.keras.layers.Dropout(self.mlp_dropout_ratio),\n                tf.keras.layers.Dense(self.enc_dim, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),\n            ]))\n\n        # Optional Projection to Decoder Dimension, new in V6\n        if self.enc_dim != self.dec_dim:\n            self.dense_out = tf.keras.layers.Dense(self.dec_dim, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False)\n            self.apply_dense_out = True\n        else:\n            self.apply_dense_out = False\n\n    def get_attention_mask(self, x_inp):\n        # Attention Mask\n        attention_mask = tf.math.count_nonzero(x_inp, axis=[2], keepdims=True, dtype=tf.int32)\n        attention_mask = tf.math.count_nonzero(attention_mask, axis=[2], keepdims=False)\n        # M5-v9 Andrew: attention_mask (B,T,S) - Mark and Hoyso are right, change back to axis=1\n        attention_mask = tf.expand_dims(attention_mask, axis=1) # axis=1\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        return attention_mask\n\n    def call(self, x, x_inp, training=False):\n        # Attention mask to ignore missing frames\n        attention_mask = self.get_attention_mask(x_inp)\n\n        # Iterate input over transformer blocks\n        for conv, ln_1, mha, ln_2, mlp in zip(self.convs, self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x = conv(x)\n            x = ln_1(x + mha(x, x, x, attention_mask=attention_mask))\n            x = ln_2(x + mlp(x))\n\n        # Optional Projection to Decoder Dimension\n        if self.apply_dense_out:\n            x = self.dense_out(x)\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:08.638812Z","iopub.execute_input":"2023-08-24T20:59:08.639633Z","iopub.status.idle":"2023-08-24T20:59:08.665809Z","shell.execute_reply.started":"2023-08-24T20:59:08.639587Z","shell.execute_reply":"2023-08-24T20:59:08.664441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modeling - decoder\n# Decoder based on multiple transformer blocks\nclass Decoder(tf.keras.Model):\n    def __init__(self, n_dec_blocks, dec_dim, n_mha_heads, mha_dropout_ratio, mlp_ratio, mlp_dropout_ratio, frame_len, phrase_len):\n        super(Decoder, self).__init__(name='decoder')\n        self.n_dec_blocks = n_dec_blocks\n        self.dec_dim = dec_dim\n        self.n_mha_heads = n_mha_heads\n        self.mha_dropout_ratio = mha_dropout_ratio\n        self.mlp_ratio = mlp_ratio\n        self.mlp_dropout_ratio = mlp_dropout_ratio\n        self.frame_len = frame_len\n        self.phrase_len = phrase_len\n        self.supports_masking = True\n\n    def build(self, input_shape):\n        # Causal Mask Batch Size 1\n        self.causal_mask = self.get_causal_attention_mask()\n        # M8-v1 not padding phrase to frame_len\n        # Positional Embedding, initialized with zeros\n        self.positional_embedding = tf.Variable(\n            initial_value=tf.zeros([self.phrase_len+1, self.dec_dim], dtype=tf.float32),\n            trainable=True,\n            name='embedding_positional_encoder',\n        )\n        # Character Embedding\n        self.char_emb = tf.keras.layers.Embedding(N_UNIQUE_CHARACTERS, self.dec_dim, embeddings_initializer=INIT_ZEROS)\n        # Positional Encoder MHA\n        # M8-v1 not padding phrase to frame_len\n        self.pos_emb_mha = MultiHeadAttention(self.dec_dim, self.n_mha_heads, self.phrase_len+1, self.phrase_len+1, self.mha_dropout_ratio)\n        self.pos_emb_ln = tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS)\n        # First Layer Normalisation\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.n_dec_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Head Attention\n            # M8-v1 not padding phrase to frame_len\n            self.mhas.append(MultiHeadAttention(self.dec_dim, self.n_mha_heads, self.phrase_len+1, self.frame_len, self.mha_dropout_ratio))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(self.dec_dim * self.mlp_ratio, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False),\n                tf.keras.layers.Dropout(self.mlp_dropout_ratio),\n                tf.keras.layers.Dense(self.dec_dim, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),\n            ]))\n\n    #M8-v1\n    def get_causal_attention_mask(self):\n        i = tf.range(self.phrase_len+1)[:, tf.newaxis]\n        j = tf.range(self.phrase_len+1)\n        mask = tf.cast(i >= j, dtype=tf.int32)\n        mask = tf.reshape(mask, (1, self.phrase_len+1, self.phrase_len+1))\n        mult = tf.concat(\n            [tf.expand_dims(1, -1), tf.constant([1, 1], dtype=tf.int32)],\n            axis=0,\n        )\n        mask = tf.tile(mask, mult)\n        mask = tf.cast(mask, tf.float32)\n        return mask\n\n    def get_attention_mask(self, x_inp):\n        # Attention Mask\n        attention_mask = tf.math.count_nonzero(x_inp, axis=[2], keepdims=True, dtype=tf.int32)\n        attention_mask = tf.math.count_nonzero(attention_mask, axis=[2], keepdims=False)\n        # M5-v9 Andrew: attention_mask (B,T,S) - Mark and Hoyso are right, change back to axis=1\n        attention_mask = tf.expand_dims(attention_mask, axis=1) # axis=1\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        return attention_mask\n\n    def call(self, encoder_outputs, phrase, x_inp, training=False):\n        # Batch Size\n        B = tf.shape(encoder_outputs)[0]\n        # Cast to INT32\n        phrase = tf.cast(phrase, tf.int32)\n        # Prepend SOS Token\n        phrase = tf.pad(phrase, [[0,0], [1,0]], constant_values=SOS_IDX, name='prepend_sos_token')\n        # Pad With PAD Token\n        # M8-v1 not padding phrase to frame_len\n        # phrase = tf.pad(phrase, [[0,0], [0,self.frame_len-self.phrase_len-1]], constant_values=PAD_IDX, name='append_pad_token')\n        # Positional Embedding\n        x = self.positional_embedding + self.char_emb(phrase)\n        # Causal Attention\n        x = self.pos_emb_ln(x + self.pos_emb_mha(x, x, x, attention_mask=self.causal_mask))\n        # Attention mask to ignore missing frames\n        attention_mask = self.get_attention_mask(x_inp)\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x = ln_1(x + mha(x, encoder_outputs, encoder_outputs, attention_mask=attention_mask))\n            x = ln_2(x + mlp(x))\n        # Slice 31 Characters\n        x = tf.slice(x, [0, 0, 0], [-1, self.phrase_len, -1])\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:08.667973Z","iopub.execute_input":"2023-08-24T20:59:08.668896Z","iopub.status.idle":"2023-08-24T20:59:08.704156Z","shell.execute_reply.started":"2023-08-24T20:59:08.668849Z","shell.execute_reply":"2023-08-24T20:59:08.702457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(frame_len=128,\n              phrase_len=32,\n              dropout_step=0,\n              n_enc_blocks=4,\n              n_dec_blocks=2,\n              enc_dim=384,\n              dec_dim=256,\n              n_mha_heads=4,\n              mha_dropout_ratio=0.2,\n              mlp_ratio=2,\n              mlp_dropout_ratio=0.3,\n              clf_dropout_ratio=0.1,\n              ):\n    # Inputs\n    frames_inp = tf.keras.layers.Input([frame_len, CHANNELS],\n                                       dtype=tf.float32,\n                                       name='frames',\n                                       )\n    phrase_inp = tf.keras.layers.Input([phrase_len],\n                                       dtype=tf.int32,\n                                       name='phrase',\n                                       )\n    # Frames\n    # x = frames_inp\n\n    # Masking, new in V6\n    x = tf.keras.layers.Masking(mask_value=PAD_FLOAT,\n                                input_shape=(frame_len, CHANNELS),\n                                name='masking',\n                                )(frames_inp) # (x)\n\n    # Embedding\n    x = Embedding(frame_len=frame_len, enc_dim=enc_dim)(x)\n\n    # Encoder Transformer Blocks\n    x = Encoder(n_enc_blocks,\n                enc_dim,\n                dec_dim,\n                n_mha_heads,\n                mha_dropout_ratio,\n                mlp_ratio,\n                mlp_dropout_ratio,\n                frame_len,\n                )(x, frames_inp)\n\n    # Decoder\n    x = Decoder(n_dec_blocks,\n                dec_dim,\n                n_mha_heads,\n                mha_dropout_ratio*1.5, # M5-v14, M8-v2\n                mlp_ratio,\n                mlp_dropout_ratio*1.5, # M5-v10,v13,v14, M8-v2\n                frame_len,\n                phrase_len,\n                )(x, phrase_inp, frames_inp)\n\n    # Classifier\n    x = tf.keras.Sequential([\n        # Dropout\n        tf.keras.layers.Dropout(clf_dropout_ratio*2),  # M5-v10,v13,v14, M8-v2\n        # Output Neurons\n        tf.keras.layers.Dense(N_UNIQUE_CHARACTERS,\n                              activation=tf.keras.activations.linear,\n                              kernel_initializer=INIT_HE_UNIFORM,\n                              use_bias=False),\n    ], name='classifier')(x)\n\n    outputs = x\n\n    # Create Tensorflow Model\n    model = tf.keras.models.Model(inputs={'frames': frames_inp, 'phrase': phrase_inp}, outputs=outputs)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:08.706153Z","iopub.execute_input":"2023-08-24T20:59:08.707749Z","iopub.status.idle":"2023-08-24T20:59:08.72328Z","shell.execute_reply.started":"2023-08-24T20:59:08.707694Z","shell.execute_reply":"2023-08-24T20:59:08.722011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    enc_dim = 384","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:08.727974Z","iopub.execute_input":"2023-08-24T20:59:08.728483Z","iopub.status.idle":"2023-08-24T20:59:08.742743Z","shell.execute_reply.started":"2023-08-24T20:59:08.728431Z","shell.execute_reply":"2023-08-24T20:59:08.741672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modeling - load weights\nmodel_path = '/kaggle/input/yu-aslfr-inference-models/aslfr-m12-v22-conv-aug-lsmooth0p25-evenmoreaug-ep200-foldall-last.h5'\nmodel = get_model(enc_dim=CFG.enc_dim)\nmodel.load_weights(model_path)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:08.744515Z","iopub.execute_input":"2023-08-24T20:59:08.745083Z","iopub.status.idle":"2023-08-24T20:59:17.799318Z","shell.execute_reply.started":"2023-08-24T20:59:08.745046Z","shell.execute_reply":"2023-08-24T20:59:17.797963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LLIP_ = [84,181,91,146,61,185,40,39,37,87,178,88,95,78,191,80,81,82]\nRLIP_ = [314,405,321,375,291,409,270,269,267,317,402,318,324,308,415,310,311,312]\nCLIP_ = [0, 17, 13, 14]\n\nLLIP_ = [f'x_face_{i}' for i in LLIP_] + [f'y_face_{i}' for i in LLIP_]\nRLIP_ = [f'x_face_{i}' for i in RLIP_] + [f'y_face_{i}' for i in RLIP_]\nCLIP_ = [f'x_face_{i}' for i in CLIP_] + [f'y_face_{i}' for i in CLIP_]\n\nLHAND_ = [f'x_left_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)]\nRHAND_ = [f'x_right_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)]\n\nLANDMARK_COLS = LLIP_ + CLIP_ +RLIP_ + LHAND_ + RHAND_\nN_COLS0 = len(LANDMARK_COLS)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:17.801279Z","iopub.execute_input":"2023-08-24T20:59:17.801657Z","iopub.status.idle":"2023-08-24T20:59:17.811824Z","shell.execute_reply.started":"2023-08-24T20:59:17.801623Z","shell.execute_reply":"2023-08-24T20:59:17.810818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TFLite model\nfor l in model.layers:\n    print(l.name)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:17.813095Z","iopub.execute_input":"2023-08-24T20:59:17.813513Z","iopub.status.idle":"2023-08-24T20:59:17.836924Z","shell.execute_reply.started":"2023-08-24T20:59:17.81348Z","shell.execute_reply":"2023-08-24T20:59:17.835339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TFLite model - preprocess\nimport numpy as np\nLIP = np.arange(0, 80).tolist()\nLLIP = np.arange(0, 36).tolist()\nCLIP = np.arange(36, 44).tolist()\nRLIP = np.arange(44, 80).tolist()\nLHAND = np.arange(80, 122).tolist()\nRHAND = np.arange(122, 164).tolist()\n\nHAND_LANDMARKS = LHAND + RHAND","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:17.839069Z","iopub.execute_input":"2023-08-24T20:59:17.839696Z","iopub.status.idle":"2023-08-24T20:59:17.853087Z","shell.execute_reply.started":"2023-08-24T20:59:17.839644Z","shell.execute_reply":"2023-08-24T20:59:17.851008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TFLite model - preprocess\ndef filter_nans_tf(x, ref_point=HAND_LANDMARKS):\n    mask = tf.math.logical_not(\n        tf.reduce_all(tf.math.is_nan(tf.gather(x, ref_point, axis=1)), axis=[-1])\n        )\n    x = tf.boolean_mask(x, mask, axis=0)\n    return x\n\ndef tf_nan_mean(x, axis=0, keepdims=False):\n    nan2zero = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n    a = tf.reduce_sum(nan2zero, axis=axis, keepdims=keepdims)\n\n    nan2zero_real2one = tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x))\n    b = tf.reduce_sum(nan2zero_real2one, axis=axis, keepdims=keepdims)\n\n    return a / b\n\ndef tf_nan_std(x, center=None, axis=0, keepdims=False):\n    if center is None:\n        center = tf_nan_mean(x, axis=axis,  keepdims=True)\n    d = x - center\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis, keepdims=keepdims))\n\nclass Preprocess(tf.keras.layers.Layer): # single seq\n    def __init__(self,\n                 max_len=128,\n                 **kwargs,\n                 ):\n        super().__init__(**kwargs)\n        self.max_len = max_len\n\n    def call(self, inputs):\n        x = filter_nans_tf(inputs)\n\n        _llip = tf.gather(x, LLIP, axis=1)\n        _clip = tf.gather(x, CLIP, axis=1)\n        _rlip = tf.gather(x, RLIP, axis=1)\n\n        _lhand = tf.gather(x, LHAND, axis=1)\n        _rhand = tf.gather(x, RHAND, axis=1)\n\n        _llip_x = _llip[:, 0*(len(LLIP)//2) : 1*(len(LLIP)//2)]\n        _llip_y = _llip[:, 1*(len(LLIP)//2) : 2*(len(LLIP)//2)]\n        _llip = tf.concat([_llip_x[..., tf.newaxis], _llip_y[..., tf.newaxis]], axis=-1)\n\n        _clip_x = _clip[:, 0*(len(CLIP)//2) : 1*(len(CLIP)//2)]\n        _clip_y = _clip[:, 1*(len(CLIP)//2) : 2*(len(CLIP)//2)]\n        _clip = tf.concat([_clip_x[..., tf.newaxis], _clip_y[..., tf.newaxis]], axis=-1)\n\n        _rlip_x = _rlip[:, 0*(len(RLIP)//2) : 1*(len(RLIP)//2)]\n        _rlip_y = _rlip[:, 1*(len(RLIP)//2) : 2*(len(RLIP)//2)]\n        _rlip = tf.concat([_rlip_x[..., tf.newaxis], _rlip_y[..., tf.newaxis]], axis=-1)\n\n        _lhand_x = _lhand[:, 0*(len(LHAND)//2) : 1*(len(LHAND)//2)]\n        _lhand_y = _lhand[:, 1*(len(LHAND)//2) : 2*(len(LHAND)//2)]\n        _lhand = tf.concat([_lhand_x[..., tf.newaxis], _lhand_y[..., tf.newaxis]], axis=-1)\n\n        _rhand_x = _rhand[:, 0*(len(RHAND)//2) : 1*(len(RHAND)//2)]\n        _rhand_y = _rhand[:, 1*(len(RHAND)//2) : 2*(len(RHAND)//2)]\n        _rhand = tf.concat([_rhand_x[..., tf.newaxis], _rhand_y[..., tf.newaxis]], axis=-1)\n\n\n        x = tf.concat([_llip, _clip, _rlip, _lhand, _rhand], axis=1)\n\n#         if augment:\n#             x = augment_fn(x, max_len=None)\n\n        center_pnts = tf.gather(x, [19], axis=1)\n        mean = tf_nan_mean(center_pnts, axis=[0, 1], keepdims=True)\n        mean = tf.where(tf.math.is_nan(mean), tf.constant(0.5,x.dtype), mean)\n        std = tf_nan_std(x, center=mean, axis=[0, 1], keepdims=True)\n        x = (x - mean) / std\n\n        # mean = tf_nan_mean(_landmarks, axis=1, keepdims=True)\n        # std = tf_nan_std(_landmarks, center=mean, axis=1, keepdims=True)\n        # x = (_landmarks - mean) / std\n\n        if self.max_len is not None:\n            if tf.shape(x)[0] < self.max_len:\n                x = tf.pad(x,\n                           paddings=[[0, self.max_len - tf.shape(x)[0]], [0, 0], [0,0]],\n                           mode='CONSTANT',\n                           constant_values=PAD_FLOAT,\n                           )\n            else:\n                x = tf.image.resize(x,\n                                    [self.max_len, tf.shape(x)[1]],\n                                    method=tf.image.ResizeMethod.BILINEAR,\n                                    )\n        length = tf.shape(x)[0]\n\n        dx1 = tf.cond(tf.shape(x)[0]>1,lambda:tf.pad(x[1:] - x[:-1], [[0,1],[0,0],[0,0]]),lambda:tf.zeros_like(x))\n\n        dx2 = tf.cond(tf.shape(x)[0]>2,lambda:tf.pad(x[2:] - x[:-2], [[0,2],[0,0],[0,0]]),lambda:tf.zeros_like(x))\n\n        x = tf.concat([\n            tf.reshape(x, (self.max_len, len(LIP)+len(HAND_LANDMARKS))),\n            tf.reshape(dx1, (self.max_len, len(LIP)+len(HAND_LANDMARKS))),\n            tf.reshape(dx2, (self.max_len, len(LIP)+len(HAND_LANDMARKS))),\n        ], axis = -1)\n\n        x = tf.where(tf.math.is_nan(x), tf.constant(0.,x.dtype), x)\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:17.855401Z","iopub.execute_input":"2023-08-24T20:59:17.856289Z","iopub.status.idle":"2023-08-24T20:59:17.899717Z","shell.execute_reply.started":"2023-08-24T20:59:17.856234Z","shell.execute_reply":"2023-08-24T20:59:17.898254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TFLite model\nMAX_PHRASE_LENGTH = 32\n\nclass TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n\n        # Load the feature generation and main models\n        self.preprocess_layer = Preprocess()\n        self.model = model\n    \n    @tf.function(jit_compile=True)\n    def encoder(self, x, frames_inp):\n        x = self.model.get_layer('embedding')(x) #, frames_inp)\n        x = self.model.get_layer('encoder')(x, frames_inp)\n        \n        return x\n        \n    @tf.function(jit_compile=True)\n    def decoder(self, x, phrase_inp, frames_inp):\n        x = self.model.get_layer('decoder')(x, phrase_inp, frames_inp)\n        x = self.model.get_layer('classifier')(x)\n        \n        return x\n    \n    @tf.function(\n        input_signature=[tf.TensorSpec(shape=[None, N_COLS0], dtype=tf.float32, name='inputs')],\n    )\n    def __call__(self, inputs):\n        # Hacky for submission\n        inputs = tf.cast(inputs, tf.float32)\n        inputs = inputs[None]\n        inputs = tf.cond(tf.shape(inputs)[1] == 0, lambda: tf.zeros((1, 1, N_COLS0)), lambda: tf.identity(inputs))\n        inputs = inputs[0]\n        # Preprocess Data\n        frames_inp = self.preprocess_layer(inputs)        \n        # Add Batch Dimension\n        frames_inp = tf.expand_dims(frames_inp, axis=0)\n        # Get Encoding\n        encoding = self.encoder(frames_inp, frames_inp)\n        # Make Prediction\n        phrase = tf.fill([1,MAX_PHRASE_LENGTH], PAD_IDX)\n        # Predict One Token At A Time\n        stop = False\n        for idx in tf.range(MAX_PHRASE_LENGTH):\n            # Cast phrase to int8\n            phrase = tf.cast(phrase, tf.int8)\n            # If EOS token is predicted, stop predicting\n            outputs = tf.cond(\n                stop,\n                lambda: tf.one_hot(tf.cast(phrase, tf.int32), N_UNIQUE_CHARACTERS),\n                lambda: self.decoder(encoding, phrase, frames_inp)\n            )\n            # Add predicted token to input phrase\n            phrase = tf.cast(phrase, tf.int32)\n            # Replace PAD token with predicted token up to idx\n            phrase = tf.where(\n                tf.range(MAX_PHRASE_LENGTH) < idx + 1,\n                tf.argmax(outputs, axis=2, output_type=tf.int32),\n                phrase,\n            )\n            # Predicted Token\n            predicted_token = phrase[0,idx]\n            # If EOS (End Of Sentence) token is predicted stop\n            if not stop:\n                stop = predicted_token == EOS_IDX\n            \n        # Squeeze outputs\n        outputs = tf.squeeze(phrase, axis=0)\n        outputs = tf.one_hot(outputs, N_UNIQUE_CHARACTERS)\n            \n        # Return a dictionary with the output tensor\n        return {'outputs': outputs }\n\n# Define TF Lite Model\ntflite_keras_model = TFLiteModel(model)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:17.901905Z","iopub.execute_input":"2023-08-24T20:59:17.902492Z","iopub.status.idle":"2023-08-24T20:59:17.926407Z","shell.execute_reply.started":"2023-08-24T20:59:17.902454Z","shell.execute_reply":"2023-08-24T20:59:17.924837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sanity check\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    CHAR2ORD = json.load(json_file)\n    \nORD2CHAR = {j:i for i,j in CHAR2ORD.items()}\n\ndef outputs2phrase(outputs):\n    if outputs.ndim == 2:\n        outputs = np.argmax(outputs, axis=1)\n    \n    return ''.join([ORD2CHAR.get(s, '') for s in outputs])","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:17.92842Z","iopub.execute_input":"2023-08-24T20:59:17.928846Z","iopub.status.idle":"2023-08-24T20:59:17.950008Z","shell.execute_reply.started":"2023-08-24T20:59:17.928812Z","shell.execute_reply":"2023-08-24T20:59:17.948618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sanity check\nimport pandas as pd\nexample_parquet_df = pd.read_parquet('/kaggle/input/asl-fingerspelling/train_landmarks/1019715464.parquet')\nfor i in range(25):\n    demo_sequence_id = example_parquet_df.index.unique()[i]\n    demo_raw_data = example_parquet_df.loc[demo_sequence_id, LANDMARK_COLS].values\n    demo_raw_data.shape\n\n    train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n    train_sequence_id = train.set_index('sequence_id')\n    demo_phrase_true = train_sequence_id.loc[demo_sequence_id, 'phrase']\n    # print(f'demo_raw_data shape: {demo_raw_data.shape}, dtype: {demo_raw_data.dtype}')\n\n    demo_output = tflite_keras_model(demo_raw_data)['outputs'].numpy()\n    # print(f'demo_output shape: {demo_output.shape}, dtype: {demo_output.dtype}')\n    print(f'demo_outputs phrase decoded: {outputs2phrase(demo_output)}')\n    print(f'phrase true: {demo_phrase_true}')","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:59:17.952027Z","iopub.execute_input":"2023-08-24T20:59:17.953111Z","iopub.status.idle":"2023-08-24T21:00:03.37506Z","shell.execute_reply.started":"2023-08-24T20:59:17.953062Z","shell.execute_reply":"2023-08-24T21:00:03.373639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission - Create Model Converter\nkeras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]#, tf.lite.OpsSet.SELECT_TF_OPS]\nkeras_model_converter.optimizations = [tf.lite.Optimize.DEFAULT]\nkeras_model_converter.target_spec.supported_types = [tf.float16]\n# Convert Model\ntflite_model = keras_model_converter.convert()\n# Write Model\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T21:00:03.378406Z","iopub.execute_input":"2023-08-24T21:00:03.380471Z","iopub.status.idle":"2023-08-24T21:02:41.90361Z","shell.execute_reply.started":"2023-08-24T21:00:03.380321Z","shell.execute_reply":"2023-08-24T21:02:41.902521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission - Add selected_columns json to only select specific columns from input frames\nwith open('inference_args.json', 'w') as f:\n     json.dump({'selected_columns': LANDMARK_COLS}, f)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T21:02:41.905283Z","iopub.execute_input":"2023-08-24T21:02:41.905896Z","iopub.status.idle":"2023-08-24T21:02:41.91201Z","shell.execute_reply.started":"2023-08-24T21:02:41.905861Z","shell.execute_reply":"2023-08-24T21:02:41.910305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission - Zip Model\n!zip submission.zip /kaggle/working/model.tflite /kaggle/working/inference_args.json","metadata":{"execution":{"iopub.status.busy":"2023-08-24T21:02:41.914803Z","iopub.execute_input":"2023-08-24T21:02:41.915432Z","iopub.status.idle":"2023-08-24T21:02:44.629867Z","shell.execute_reply.started":"2023-08-24T21:02:41.915382Z","shell.execute_reply":"2023-08-24T21:02:44.627846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluation\n# def load_relevant_data_subset(pq_path):\n#     return pd.read_parquet(pq_path, columns=LANDMARK_COLS)\n\n# pq_path = '/kaggle/input/asl-fingerspelling/train_landmarks/1019715464.parquet'\n# parquet_df = load_relevant_data_subset(pq_path)\n# sequence_id = example_parquet_df.index.unique()[0]\n# frames = parquet_df.loc[demo_sequence_id].values","metadata":{"execution":{"iopub.status.busy":"2023-08-24T21:02:44.632972Z","iopub.execute_input":"2023-08-24T21:02:44.633615Z","iopub.status.idle":"2023-08-24T21:02:44.641785Z","shell.execute_reply.started":"2023-08-24T21:02:44.633555Z","shell.execute_reply":"2023-08-24T21:02:44.64078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluation\n# !pip install tflite-runtime==2.13.0","metadata":{"execution":{"iopub.status.busy":"2023-08-24T21:02:44.643967Z","iopub.execute_input":"2023-08-24T21:02:44.644541Z","iopub.status.idle":"2023-08-24T21:02:44.656261Z","shell.execute_reply.started":"2023-08-24T21:02:44.64449Z","shell.execute_reply":"2023-08-24T21:02:44.654786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Evaluation\n# import tflite_runtime.interpreter as tflite\n# interpreter = tflite.Interpreter(model_path)\n\n# REQUIRED_SIGNATURE = \"serving_default\"\n# REQUIRED_OUTPUT = \"outputs\"\n\n# with open (\"/kaggle/input/fingerspelling-character-map/character_to_prediction_index.json\", \"r\") as f:\n#     character_map = json.load(f)\n# rev_character_map = {j:i for i,j in character_map.items()}\n\n# found_signatures = list(interpreter.get_signature_list().keys())\n\n# if REQUIRED_SIGNATURE not in found_signatures:\n#     raise KernelEvalException('Required input signature not found.')\n\n# prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n# output = prediction_fn(inputs=frames)\n# prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])","metadata":{"execution":{"iopub.status.busy":"2023-08-24T21:02:44.658733Z","iopub.execute_input":"2023-08-24T21:02:44.659292Z","iopub.status.idle":"2023-08-24T21:02:44.671218Z","shell.execute_reply.started":"2023-08-24T21:02:44.659254Z","shell.execute_reply":"2023-08-24T21:02:44.670037Z"},"trusted":true},"execution_count":null,"outputs":[]}]}