{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport json\nimport math\nimport pickle\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow_addons as tfa","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:36.097677Z","iopub.execute_input":"2023-08-24T18:56:36.098216Z","iopub.status.idle":"2023-08-24T18:56:36.105165Z","shell.execute_reply.started":"2023-08-24T18:56:36.098183Z","shell.execute_reply":"2023-08-24T18:56:36.103142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\npad_token = '^'\npad_token_idx = 59\n\nchar_to_num[pad_token] = pad_token_idx\n\nnum_to_char = {j:i for i,j in char_to_num.items()}\ndf = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n\nLIP = [\n    61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\n# #new face_id\nface_id = [454,356,323,361,389,288,251,264,447,366,368,\n           401,397,435,284,301,372,345,383,367,365,352,433,\n           376,298,265,93,234,300,132,340,353,127]\n# LIP = list(set(LIP + face_id))\nfor k in face_id:\n    if k not in LIP:\n        LIP.append(k)\nl = len(LIP)\n\nLIP  = LIP[:int(l-l/4)]\n# LPOSE = [13, 15, 17, 19, 21]\n# RPOSE = [14, 16, 18, 20, 22]\nLPOSE = [1,3,5,7,9,11,13, 15,17, 19, 21,23,25,27,29,31]#\nRPOSE = [0,2,4,6,8,10,12, 14,16, 18, 20,22,24,26,28,30]#\nPOSE = LPOSE + RPOSE\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE] + [f'x_face_{i}' for i in LIP]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE] + [f'y_face_{i}' for i in LIP]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE] + [f'z_face_{i}' for i in LIP]\n\nSEL_COLS = X + Y + Z\n# Switch fream_len to 256\nFRAME_LEN = 256#128\nMAX_PHRASE_LENGTH = 64\n\nLIP_IDX_X   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"x\" in col]\nRHAND_IDX_X = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"x\" in col]\nLHAND_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"x\" in col]\nRPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in RPOSE and \"x\" in col]\nLPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in LPOSE and \"x\" in col]\n\nLIP_IDX_Y   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"y\" in col]\nRHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"y\" in col]\nLHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"y\" in col]\nRPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in RPOSE and \"y\" in col]\nLPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in LPOSE and \"y\" in col]\n\nLIP_IDX_Z   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"z\" in col]\nRHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"z\" in col]\nLHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"z\" in col]\nRPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in RPOSE and \"z\" in col]\nLPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in LPOSE and \"z\" in col]\n\nRHM = np.load(\"/kaggle/input/new-coordinate-mean-std/rh_mean.npy\")\nLHM = np.load(\"/kaggle/input/new-coordinate-mean-std/lh_mean.npy\")\nRPM = np.load(\"/kaggle/input/new-coordinate-mean-std/rp_mean.npy\")\nLPM = np.load(\"/kaggle/input/new-coordinate-mean-std/lp_mean.npy\")\nLIPM = np.load(\"/kaggle/input/new-coordinate-mean-std/lip_mean.npy\")\n\nRHS = np.load(\"/kaggle/input/new-coordinate-mean-std/rh_std.npy\")\nLHS = np.load(\"/kaggle/input/new-coordinate-mean-std/lh_std.npy\")\nRPS = np.load(\"/kaggle/input/new-coordinate-mean-std/rp_std.npy\")\nLPS = np.load(\"/kaggle/input/new-coordinate-mean-std/lp_std.npy\")\nLIPS = np.load(\"/kaggle/input/new-coordinate-mean-std/lip_std.npy\")\nprint(len(LIP))\nprint(RHM.shape,LHM.shape)\n","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:36.112274Z","iopub.execute_input":"2023-08-24T18:56:36.113675Z","iopub.status.idle":"2023-08-24T18:56:36.266803Z","shell.execute_reply.started":"2023-08-24T18:56:36.113625Z","shell.execute_reply":"2023-08-24T18:56:36.265679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=SEL_COLS)\n\nfile_id = df.file_id.iloc[0]\ninpdir = \"/kaggle/input/asl-fingerspelling/train_landmarks\"\npqfile = f\"{inpdir}/{file_id}.parquet\"\nseq_refs = df.loc[df.file_id == file_id]\nseqs = load_relevant_data_subset(pqfile)\n\nseq_id = seq_refs.sequence_id.iloc[0]\nframes = seqs.iloc[seqs.index == seq_id]\nphrase = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])\nphrase","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:36.268597Z","iopub.execute_input":"2023-08-24T18:56:36.269017Z","iopub.status.idle":"2023-08-24T18:56:36.867967Z","shell.execute_reply.started":"2023-08-24T18:56:36.268987Z","shell.execute_reply":"2023-08-24T18:56:36.867188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# motion feature","metadata":{}},{"cell_type":"code","source":"@tf.function()\ndef resize_pad(x):\n    if tf.shape(x)[0] < FRAME_LEN:\n        x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]), constant_values=float(\"NaN\"))\n    else:\n        x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n    return x\n\n@tf.function(jit_compile=True)\ndef pre_process0(x):\n    lip_x = tf.gather(x, LIP_IDX_X, axis=1)\n    lip_y = tf.gather(x, LIP_IDX_Y, axis=1)\n    lip_z = tf.gather(x, LIP_IDX_Z, axis=1)\n\n    rhand_x = tf.gather(x, RHAND_IDX_X, axis=1)\n    rhand_y = tf.gather(x, RHAND_IDX_Y, axis=1)\n    rhand_z = tf.gather(x, RHAND_IDX_Z, axis=1)\n\n    lhand_x = tf.gather(x, LHAND_IDX_X, axis=1)\n    lhand_y = tf.gather(x, LHAND_IDX_Y, axis=1)\n    lhand_z = tf.gather(x, LHAND_IDX_Z, axis=1)\n\n    rpose_x = tf.gather(x, RPOSE_IDX_X, axis=1)\n    rpose_y = tf.gather(x, RPOSE_IDX_Y, axis=1)\n    rpose_z = tf.gather(x, RPOSE_IDX_Z, axis=1)\n\n    lpose_x = tf.gather(x, LPOSE_IDX_X, axis=1)\n    lpose_y = tf.gather(x, LPOSE_IDX_Y, axis=1)\n    lpose_z = tf.gather(x, LPOSE_IDX_Z, axis=1)\n\n    lip   = tf.concat([lip_x[..., tf.newaxis], lip_y[..., tf.newaxis], lip_z[..., tf.newaxis]], axis=-1)\n    rhand = tf.concat([rhand_x[..., tf.newaxis], rhand_y[..., tf.newaxis], rhand_z[..., tf.newaxis]], axis=-1)\n    lhand = tf.concat([lhand_x[..., tf.newaxis], lhand_y[..., tf.newaxis], lhand_z[..., tf.newaxis]], axis=-1)\n    rpose = tf.concat([rpose_x[..., tf.newaxis], rpose_y[..., tf.newaxis], rpose_z[..., tf.newaxis]], axis=-1)\n    lpose = tf.concat([lpose_x[..., tf.newaxis], lpose_y[..., tf.newaxis], lpose_z[..., tf.newaxis]], axis=-1)\n\n    hand = tf.concat([rhand, lhand,lip,rpose,lpose], axis=1)\n    hand = tf.where(tf.math.is_nan(hand), 0.0, hand)\n    mask = tf.math.not_equal(tf.reduce_sum(hand, axis=[1, 2]), 0.0)\n\n    lip = lip[mask]\n    rhand = rhand[mask]\n    lhand = lhand[mask]\n    rpose = rpose[mask]\n    lpose = lpose[mask]\n\n    return lip, rhand, lhand, rpose, lpose\n\n@tf.function()\ndef pre_process1(lip, rhand, lhand, rpose, lpose):\n    lip   = (resize_pad(lip) - LIPM) / LIPS\n    rhand = (resize_pad(rhand) - RHM) / RHS\n    lhand = (resize_pad(lhand) - LHM) / LHS\n    rpose = (resize_pad(rpose) - RPM) / RPS\n    lpose = (resize_pad(lpose) - LPM) / LPS\n\n    x = tf.concat([lip, rhand, lhand, rpose, lpose], axis=1)\n    ##motionfeature\n    length = tf.shape(x)[0]\n    dx = tf.cond(tf.shape(x)[0]>1,lambda:tf.pad(x[1:] - x[:-1], [[0,1],[0,0],[0,0]]),lambda:tf.zeros_like(x))\n    dx2 = tf.cond(tf.shape(x)[0]>2,lambda:tf.pad(x[2:] - x[:-2], [[0,2],[0,0],[0,0]]),lambda:tf.zeros_like(x))\n    x = tf.concat([x,dx,dx2], axis = -1)\n    s = tf.shape(x)\n    x = tf.reshape(x, (s[0], s[1]*s[2]))\n    x = tf.where(tf.math.is_nan(x), 0.0, x)\n    return x\n\n\npre0 = pre_process0(frames)\npre1 = pre_process1(*pre0)\nINPUT_SHAPE = list(pre1.shape)\nprint(INPUT_SHAPE)\nzero_count = tf.reduce_sum(tf.cast(tf.equal(pre1, 0), tf.int32))\n# Convert to Python integer if needed\nzero_count = zero_count.numpy()\nprint(zero_count)  ","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:36.869058Z","iopub.execute_input":"2023-08-24T18:56:36.869922Z","iopub.status.idle":"2023-08-24T18:56:40.640162Z","shell.execute_reply.started":"2023-08-24T18:56:36.86988Z","shell.execute_reply":"2023-08-24T18:56:40.639091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Record Val_len and batch size\n","metadata":{}},{"cell_type":"code","source":"def decode_fn(record_bytes):\n    schema = {\n        \"lip\": tf.io.VarLenFeature(tf.float32),\n        \"rhand\": tf.io.VarLenFeature(tf.float32),\n        \"lhand\": tf.io.VarLenFeature(tf.float32),\n        \"rpose\": tf.io.VarLenFeature(tf.float32),\n        \"lpose\": tf.io.VarLenFeature(tf.float32),\n        \"phrase\": tf.io.VarLenFeature(tf.int64)\n    }\n    x = tf.io.parse_single_example(record_bytes, schema)\n\n    lip = tf.reshape(tf.sparse.to_dense(x[\"lip\"]), (-1, 54, 3))\n    rhand = tf.reshape(tf.sparse.to_dense(x[\"rhand\"]), (-1, 21, 3))\n    lhand = tf.reshape(tf.sparse.to_dense(x[\"lhand\"]), (-1, 21, 3))\n    rpose = tf.reshape(tf.sparse.to_dense(x[\"rpose\"]), (-1, 16, 3))\n    lpose = tf.reshape(tf.sparse.to_dense(x[\"lpose\"]), (-1, 16, 3))\n    phrase = tf.sparse.to_dense(x[\"phrase\"])\n\n    return lip, rhand, lhand, rpose, lpose, phrase\n\ndef pre_process_fn(lip, rhand, lhand, rpose, lpose, phrase):\n    phrase = tf.pad(phrase, [[0, MAX_PHRASE_LENGTH-tf.shape(phrase)[0]]], constant_values=pad_token_idx)\n    return pre_process1(lip, rhand, lhand, rpose, lpose), phrase","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:40.641709Z","iopub.execute_input":"2023-08-24T18:56:40.642224Z","iopub.status.idle":"2023-08-24T18:56:40.653535Z","shell.execute_reply.started":"2023-08-24T18:56:40.642183Z","shell.execute_reply":"2023-08-24T18:56:40.652287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model config","metadata":{}},{"cell_type":"code","source":"class ECA(tf.keras.layers.Layer):\n    def __init__(self, kernel_size=5, **kwargs):\n        super().__init__(**kwargs)\n        self.supports_masking = True\n        self.kernel_size = kernel_size\n        self.conv = tf.keras.layers.Conv1D(1, kernel_size=kernel_size, strides=1, padding=\"same\", use_bias=False)\n\n    def call(self, inputs, mask=None):\n        nn = tf.keras.layers.GlobalAveragePooling1D()(inputs, mask=mask)\n        nn = tf.expand_dims(nn, -1)\n        nn = self.conv(nn)\n        nn = tf.squeeze(nn, -1)\n        nn = tf.nn.sigmoid(nn)\n        nn = nn[:,None,:]\n        return inputs * nn\n\nclass CausalDWConv1D(tf.keras.layers.Layer):\n    def __init__(self,\n        kernel_size=17,\n        dilation_rate=1,\n        use_bias=False,\n        depthwise_initializer='glorot_uniform',\n        name='', **kwargs):\n        super().__init__(name=name,**kwargs)\n        self.causal_pad = tf.keras.layers.ZeroPadding1D((dilation_rate*(kernel_size-1),0),name=name + '_pad')\n        self.dw_conv = tf.keras.layers.DepthwiseConv1D(\n                            kernel_size,\n                            strides=1,\n                            dilation_rate=dilation_rate,\n                            padding='valid',\n                            use_bias=use_bias,\n                            depthwise_initializer=depthwise_initializer,\n                            name=name + '_dwconv')\n        self.supports_masking = True\n\n    def call(self, inputs):\n        x = self.causal_pad(inputs)\n        x = self.dw_conv(x)\n        return x\n\ndef Conv1DBlock(channel_size,\n          kernel_size,\n          dilation_rate=1,\n          drop_rate=0.0,\n          expand_ratio=2,\n          se_ratio=0.25,\n          activation='swish',\n          name=None):\n    '''\n    efficient conv1d block, @hoyso48\n    '''\n    if name is None:\n        name = str(tf.keras.backend.get_uid(\"mbblock\"))\n    # Expansion phase\n    def apply(inputs):\n        channels_in = tf.keras.backend.int_shape(inputs)[-1]\n        channels_expand = channels_in * expand_ratio\n\n        skip = inputs\n\n        x = tf.keras.layers.Dense(\n            channels_expand,\n            use_bias=True,\n            activation=activation,\n            name=name + '_expand_conv')(inputs)\n\n        # Depthwise Convolution\n        x = CausalDWConv1D(kernel_size,\n            dilation_rate=dilation_rate,\n            use_bias=False,\n            name=name + '_dwconv')(x)\n\n        x = tf.keras.layers.BatchNormalization(momentum=0.95, name=name + '_bn')(x)\n\n        x  = ECA()(x)\n\n        x = tf.keras.layers.Dense(\n            channel_size,\n            use_bias=True,\n            name=name + '_project_conv')(x)\n\n        if drop_rate > 0:\n            x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1), name=name + '_drop')(x)\n\n        #保留残差网络\n        # if (channels_in == channel_size):\n        x = tf.keras.layers.add([x, skip], name=name + '_add')\n        return x\n\n    return apply\n\nclass MultiHeadSelfAttention(tf.keras.layers.Layer):\n    def __init__(self, dim=256, num_heads=4, dropout=0, **kwargs):\n        super().__init__(**kwargs)\n        self.dim = dim\n        self.scale = self.dim ** -0.5\n        self.num_heads = num_heads\n        self.qkv = tf.keras.layers.Dense(3 * dim, use_bias=False)\n        self.drop1 = tf.keras.layers.Dropout(dropout)\n        self.proj = tf.keras.layers.Dense(dim, use_bias=False)\n        self.supports_masking = True\n\n    def call(self, inputs, mask=None):\n        qkv = self.qkv(inputs)\n        qkv = tf.keras.layers.Permute((2, 1, 3))(tf.keras.layers.Reshape((-1, self.num_heads, self.dim * 3 // self.num_heads))(qkv))\n        q, k, v = tf.split(qkv, [self.dim // self.num_heads] * 3, axis=-1)\n\n        attn = tf.matmul(q, k, transpose_b=True) * self.scale\n\n        if mask is not None:\n            mask = mask[:, None, None, :]\n\n        attn = tf.keras.layers.Softmax(axis=-1)(attn, mask=mask)\n        attn = self.drop1(attn)\n\n        x = attn @ v\n        x = tf.keras.layers.Reshape((-1, self.dim))(tf.keras.layers.Permute((2, 1, 3))(x))\n        x = self.proj(x)\n        return x\n\n\ndef TransformerBlock(dim=256, num_heads=6, expand=4, attn_dropout=0.2, drop_rate=0.2, activation='swish'):\n    def apply(inputs):\n        x = inputs\n        x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n        x = MultiHeadSelfAttention(dim=dim,num_heads=num_heads,dropout=attn_dropout)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([inputs, x])\n        attn_out = x\n\n        x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n        x = tf.keras.layers.Dense(dim*expand, use_bias=False, activation=activation)(x)\n        x = tf.keras.layers.Dense(dim, use_bias=False)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([attn_out, x])\n        return x\n    return apply\n\ndef positional_encoding(maxlen, num_hid):\n        depth = num_hid/2\n        positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n        depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n        angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n        angle_rads = tf.linalg.matmul(positions, angle_rates)\n        pos_encoding = tf.concat(\n          [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n          axis=-1)\n        return pos_encoding","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:40.656388Z","iopub.execute_input":"2023-08-24T18:56:40.656698Z","iopub.status.idle":"2023-08-24T18:56:40.688583Z","shell.execute_reply.started":"2023-08-24T18:56:40.656672Z","shell.execute_reply":"2023-08-24T18:56:40.687323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# landmark Embedding","metadata":{}},{"cell_type":"code","source":"# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu\n\nLHAND_IDX = LHAND_IDX_X + LHAND_IDX_Y + LHAND_IDX_Z\nRHAND_IDX = RHAND_IDX_X + RHAND_IDX_Y + RHAND_IDX_Z\nLIP_IDX = LIP_IDX_X + LIP_IDX_Y + LIP_IDX_Z\nPOSE_IDX = LPOSE_IDX_X + RPOSE_IDX_X + LPOSE_IDX_Y + RPOSE_IDX_Y + LPOSE_IDX_Z + RPOSE_IDX_Z\n\n# Landmark Embedding\n# Embeds a landmark using local features and global features\nclass LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name='landmark_embedding'):\n        super(LandmarkEmbedding, self).__init__(name=name)\n        self.units = units\n        self.supports_masking = True\n        \n    def build(self, input_shape):\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # local feature extractor\n        self.lefthand_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_lefthand', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU)\n        self.righthand_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_righthand', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU)\n        self.lips_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_lips', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU)\n        self.pose_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_pose', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU)\n\n        # full feature extractor\n        self.full_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_full', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU)\n        \n        # global feature extractor\n        self.global_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_global', use_bias=False, kernel_initializer=INIT_HE_UNIFORM)\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.global_mlp(tf.concat([self.full_extractor(x), self.local_extractor(x)], axis=-1))\n            )\n    \n    def local_extractor(self, x):\n\n        lefthand_feature = self.lefthand_mlp(tf.gather(x, LHAND_IDX, axis=-1))\n        righthad_featrue = self.righthand_mlp(tf.gather(x, RHAND_IDX, axis=-1))\n        # Keeps dominant hand feature\n        hand_feature = tf.reduce_max(tf.stack([lefthand_feature, righthad_featrue], axis=-1), axis=-1)\n        lips_featrue = self.lips_mlp(tf.gather(x, LIP_IDX, axis=-1))\n        pose_featrue = self.pose_mlp(tf.gather(x, POSE_IDX, axis=-1))\n        \n        return tf.concat([hand_feature, lips_featrue, pose_featrue], axis=-1)\n    \n    def full_extractor(self, x):\n        return self.full_mlp(x)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:40.6903Z","iopub.execute_input":"2023-08-24T18:56:40.690682Z","iopub.status.idle":"2023-08-24T18:56:40.708751Z","shell.execute_reply.started":"2023-08-24T18:56:40.69065Z","shell.execute_reply":"2023-08-24T18:56:40.707724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# check num_hand and dropout for transfomer","metadata":{}},{"cell_type":"code","source":"def CTCLoss(labels, logits):\n    label_length = tf.reduce_sum(tf.cast(labels != pad_token_idx, tf.int32), axis=-1)\n    logit_length = tf.ones(tf.shape(logits)[0], dtype=tf.int32) * tf.shape(logits)[1]\n    loss = tf.nn.ctc_loss(\n            labels=labels,\n            logits=logits,\n            label_length=label_length,\n            logit_length=logit_length,\n            blank_index=pad_token_idx,\n            logits_time_major=False\n        )\n    loss = tf.reduce_mean(loss)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:40.710644Z","iopub.execute_input":"2023-08-24T18:56:40.711054Z","iopub.status.idle":"2023-08-24T18:56:40.724978Z","shell.execute_reply.started":"2023-08-24T18:56:40.711023Z","shell.execute_reply":"2023-08-24T18:56:40.723961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def smooth_ctc_loss(labels, logits, num_classes = 60 , blank=0, weight=0.7):\n    # Compute CTC Loss\n    label_length = tf.reduce_sum(tf.cast(labels != pad_token_idx, tf.int32), axis=-1)\n    logit_length = tf.ones(tf.shape(logits)[0], dtype=tf.int32) * tf.shape(logits)[1]\n    ctc_loss = tf.nn.ctc_loss(\n        labels=labels,\n        logits=logits,\n        label_length=label_length,\n        logit_length=logit_length,\n        blank_index=pad_token_idx,\n        logits_time_major=False\n    )\n    ctc_loss = tf.reduce_mean(ctc_loss)\n\n    # Compute KL Divergence Loss\n    kl_inp = tf.nn.softmax(logits)\n\n    # Create the target distribution\n    kl_tar = tf.fill(tf.shape(logits), 1. / num_classes)\n\n    # Compute the KL divergence\n    kldiv_loss = (tf.keras.losses.KLDivergence(tf.keras.losses.Reduction.NONE)(kl_tar, kl_inp) \n                 + tf.keras.losses.KLDivergence(tf.keras.losses.Reduction.NONE)(kl_inp, kl_tar))/2.0\n\n    kldiv_loss = tf.reduce_mean(kldiv_loss)\n\n    # Combined Loss\n    loss = (1. - weight) * ctc_loss + weight * kldiv_loss\n    #loss = ctc_loss\n    #loss = kldiv_loss\n    return loss\n    return combined_loss","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:40.726351Z","iopub.execute_input":"2023-08-24T18:56:40.726678Z","iopub.status.idle":"2023-08-24T18:56:40.740135Z","shell.execute_reply.started":"2023-08-24T18:56:40.726643Z","shell.execute_reply":"2023-08-24T18:56:40.73921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get_model","metadata":{}},{"cell_type":"code","source":"def get_model(dim = 384,num_blocks = 6,drop_rate = 0.4): \n    inp = tf.keras.Input(INPUT_SHAPE)\n    x = inp\n    #x = tf.keras.layers.Masking(mask_value=0.0)(inp)\n      # original\n    x = tf.keras.layers.Dense(dim, use_bias=False, name='stem_conv')(x)\n      # local & global extractor\n    #x = LandmarkEmbedding(dim)(x)\n    pe = tf.cast(positional_encoding(INPUT_SHAPE[0], dim), dtype=x.dtype)\n    x = x + pe\n    x = tf.keras.layers.BatchNormalization(momentum=0.95,name='stem_bn')(x)\n\n    for i in range(num_blocks):\n        x = Conv1DBlock(dim, 11, drop_rate=drop_rate)(x)\n        x = Conv1DBlock(dim,  5, drop_rate=drop_rate)(x)\n        x = Conv1DBlock(dim,  3, drop_rate=drop_rate)(x)\n        x = TransformerBlock(dim, expand=2)(x)\n    \n    x = tf.keras.layers.Dense(dim*2,tf.keras.activations.gelu,name='top_conv')(x)\n    x = tf.keras.layers.Dropout(0.4)(x)\n    x = tf.keras.layers.Dense(len(char_to_num),name='classifier')(x)\n\n    model = tf.keras.Model(inp, x)\n\n    loss = smooth_ctc_loss\n\n   # Adam Optimizer\n    optimizer = tfa.optimizers.RectifiedAdam(sma_threshold=4)\n    optimizer = tfa.optimizers.Lookahead(optimizer, sync_period=5)\n\n    model.compile(loss=loss, optimizer=optimizer)\n\n    return model\n\n\ntf.keras.backend.clear_session()\nmodel = get_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:40.74151Z","iopub.execute_input":"2023-08-24T18:56:40.741837Z","iopub.status.idle":"2023-08-24T18:56:44.617104Z","shell.execute_reply.started":"2023-08-24T18:56:40.741811Z","shell.execute_reply":"2023-08-24T18:56:44.615982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def num_to_char_fn(y):\n    return [num_to_char.get(x, \"\") for x in y]\n\n@tf.function()\ndef decode_phrase(pred):\n    x = tf.argmax(pred, axis=1)\n    diff = tf.not_equal(x[:-1], x[1:])\n    adjacent_indices = tf.where(diff)[:, 0]\n    x = tf.gather(x, adjacent_indices)\n    mask = x != pad_token_idx\n    x = tf.boolean_mask(x, mask, axis=0)\n    return x\n\n# A utility function to decode the output of the network\ndef decode_batch_predictions(pred):\n    output_text = []\n    for result in pred:\n        result = \"\".join(num_to_char_fn(decode_phrase(result).numpy()))\n        output_text.append(result)\n    return output_text","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:44.618681Z","iopub.execute_input":"2023-08-24T18:56:44.619188Z","iopub.status.idle":"2023-08-24T18:56:44.63094Z","shell.execute_reply.started":"2023-08-24T18:56:44.619148Z","shell.execute_reply":"2023-08-24T18:56:44.62963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# record training parameter","metadata":{}},{"cell_type":"code","source":"N_EPOCHS = 120\nN_WARMUP_EPOCHS = 10\nLR_MAX = 1e-3*0.5\nWD_RATIO = 0.05\nWARMUP_METHOD = \"exp\"","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:44.632585Z","iopub.execute_input":"2023-08-24T18:56:44.633522Z","iopub.status.idle":"2023-08-24T18:56:44.642756Z","shell.execute_reply.started":"2023-08-24T18:56:44.633488Z","shell.execute_reply":"2023-08-24T18:56:44.641519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2080/2080 - 147s - loss: 5.9612 - val_loss: 7.7208 - lr: 1.9173e-06 - 147s/epoch - 71ms/step\n# learning rate: 8.53e-07, weight decay: 4.26e-08\n# Epoch 99/100\n# Lev distance: 0.8002586286240221","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:44.644308Z","iopub.execute_input":"2023-08-24T18:56:44.644629Z","iopub.status.idle":"2023-08-24T18:56:44.6557Z","shell.execute_reply.started":"2023-08-24T18:56:44.644602Z","shell.execute_reply":"2023-08-24T18:56:44.654779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.save_weights('model_test.h5')","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:44.657249Z","iopub.execute_input":"2023-08-24T18:56:44.657542Z","iopub.status.idle":"2023-08-24T18:56:44.671001Z","shell.execute_reply.started":"2023-08-24T18:56:44.657517Z","shell.execute_reply":"2023-08-24T18:56:44.669969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# load weight","metadata":{}},{"cell_type":"code","source":"# Load Weights\nLOAD_WEIGHTS = True\nif LOAD_WEIGHTS:\n    print('yes')\n    model.load_weights('/kaggle/input/final-epoch80/model_epoch_80.h5')\n    print(f'Successfully Loaded Pretrained Weights')","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:44.67429Z","iopub.execute_input":"2023-08-24T18:56:44.674643Z","iopub.status.idle":"2023-08-24T18:56:46.808114Z","shell.execute_reply.started":"2023-08-24T18:56:44.674614Z","shell.execute_reply":"2023-08-24T18:56:46.807166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# check process1  fn","metadata":{}},{"cell_type":"code","source":"class TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n        self.model = model\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(SEL_COLS)], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs, training=False):\n        # Preprocess Data\n        x = tf.cast(inputs, tf.float32)\n        x = x[None]\n        x = tf.cond(tf.shape(x)[1] == 0, lambda: tf.zeros((1, 1, len(SEL_COLS))), lambda: tf.identity(x))\n        x = x[0]\n        x = pre_process0(x)\n        x = pre_process1(*x)\n        x = tf.reshape(x, INPUT_SHAPE)\n        x = x[None]\n        x = self.model(x, training=False)\n        x = x[0]\n        x = decode_phrase(x)\n        x = tf.cond(tf.shape(x)[0] == 0, lambda: tf.zeros(1, tf.int64), lambda: tf.identity(x))\n        x = tf.one_hot(x, 59)\n        return {'outputs': x}\n\ntflitemodel_base = TFLiteModel(model)\ntflitemodel_base(frames)[\"outputs\"].shape","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:46.809597Z","iopub.execute_input":"2023-08-24T18:56:46.809867Z","iopub.status.idle":"2023-08-24T18:56:53.014759Z","shell.execute_reply.started":"2023-08-24T18:56:46.809843Z","shell.execute_reply":"2023-08-24T18:56:53.013975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]#, tf.lite.OpsSet.SELECT_TF_OPS]\nkeras_model_converter.optimizations = [tf.lite.Optimize.DEFAULT]\nkeras_model_converter.target_spec.supported_types = [tf.float16]\ntflite_model = keras_model_converter.convert()\nwith open('model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \nwith open('inference_args.json', \"w\") as f:\n    json.dump({\"selected_columns\" : SEL_COLS}, f)\n    \n!zip submission.zip  './model.tflite' './inference_args.json'","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:56:53.016955Z","iopub.execute_input":"2023-08-24T18:56:53.017263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"inference_args.json\", \"r\") as f:\n    SEL_COLS = json.load(f)[\"selected_columns\"]\n    \ndef load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=SEL_COLS)\n\ndef create_data_gen(file_ids, y_mul=1):\n    def gen():\n        for file_id in file_ids:\n            pqfile = f\"{inpdir}/{file_id}.parquet\"\n            seq_refs = df.loc[df.file_id == file_id]\n            seqs = load_relevant_data_subset(pqfile)\n            for seq_id in seq_refs.sequence_id:\n                x = seqs.iloc[seqs.index == seq_id].to_numpy()\n                y = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])\n                yield x, y\n    return gen\n\npqfiles = df.file_id.unique()\nval_len = int(0.05 * len(pqfiles))\n\ntest_dataset = tf.data.Dataset.from_generator(create_data_gen(pqfiles[:val_len], 0),\n    output_signature=(tf.TensorSpec(shape=(None, len(SEL_COLS)), dtype=tf.float32), tf.TensorSpec(shape=(), dtype=tf.string))\n).prefetch(buffer_size=2000)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tf.lite.Interpreter(\"model.tflite\")\n\nREQUIRED_SIGNATURE = \"serving_default\"\nREQUIRED_OUTPUT = \"outputs\"\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nprediction_fn = interpreter.get_signature_runner(REQUIRED_SIGNATURE)\n\nfor frame, target in test_dataset.skip(300).take(10):\n    output = prediction_fn(inputs=frame)\n    prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\n    target = target.numpy().decode(\"utf-8\")\n    print(\"predic =\", prediction_str) \n    print(\"target =\", target)\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%timeit -n 10\n# output = prediction_fn(inputs=frame)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Testing = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from Levenshtein import distance\nimport heapq\nLD_Val = []\nN = 200\nif Testing:\n    for i, (frame, target) in tqdm(enumerate(test_dataset.take(N))):\n        output = prediction_fn(inputs=frame)\n        prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\n        target = target.numpy().decode(\"utf-8\")\n        lev_distance= distance(prediction_str, target)\n        LD_Val.append(lev_distance)\n    hashmap = dict()\n    for lav in LD_Val:\n        if lav not in hashmap:\n            hashmap[lav] =1\n        else:\n            hashmap[lav] +=1\n    plt.figure(figsize = (15,8))\n    plt.bar(hashmap.keys(),hashmap.values())\n    plt.show()\n\n    pri_que = []\n    for key, value in hashmap.items():\n        heapq.heappush(pri_que,(value,key))\n        if len(pri_que) > 6:\n            heapq.heappop(pri_que)\n    res = [0] * 6\n    for k in range(len(res)-1,-1,-1):\n        res[k]=heapq.heappop(pri_que)\n    print(res)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scores = []\n\n# for i, (frame, target) in tqdm(enumerate(test_dataset.take(5000))):\n#     output = prediction_fn(inputs=frame)\n#     prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\n#     target = target.numpy().decode(\"utf-8\")\n#     score = (len(target) - distance(prediction_str, target)) / len(target)\n#     scores.append(score)\n#     if i % 500 == 0:\n#         print(np.sum(scores) / len(scores))\n    \n# scores = np.array(scores)\n# print(np.sum(scores) / len(scores))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}