{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom skimage.transform import resize\nfrom sklearn.model_selection import train_test_split\nimport json\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:55:30.306487Z","iopub.execute_input":"2023-06-11T09:55:30.306834Z","iopub.status.idle":"2023-06-11T09:55:37.910511Z","shell.execute_reply.started":"2023-06-11T09:55:30.306806Z","shell.execute_reply":"2023-06-11T09:55:37.909406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\nnum_to_char = {j:i for i,j in char_to_num.items()}\n\ninpdir = \"/kaggle/input/asl-fingerspelling\"\ndf = pd.read_csv(f'{inpdir}/train.csv')\n\nLIP = [\n    61, 185, 40, 39, 37, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\n\nFACE = [f'x_face_{i}' for i in LIP] + [f'y_face_{i}' for i in LIP] + [f'z_face_{i}' for i in LIP]\nLHAND = [f'x_left_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)]\nRHAND = [f'x_right_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)] + [f'z_right_hand_{i}' for i in range(21)]\nPOSE = [f'x_pose_{i}' for i in range(33)] + [f'y_pose_{i}' for i in range(33)] + [f'z_pose_{i}' for i in range(33)]\n\nX = [f'x_face_{i}' for i in LIP] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_right_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in range(33)]\nY = [f'y_face_{i}' for i in LIP] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in range(33)]\nZ = [f'z_face_{i}' for i in LIP] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_right_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in range(33)]\n\n#SEL_COLS = FACE + LHAND + RHAND + POSE\nSEL_COLS = X + Y + Z\nFRAME_LEN = 128","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:55:37.912822Z","iopub.execute_input":"2023-06-11T09:55:37.913663Z","iopub.status.idle":"2023-06-11T09:55:38.069601Z","shell.execute_reply.started":"2023-06-11T09:55:37.913625Z","shell.execute_reply":"2023-06-11T09:55:38.068597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(SEL_COLS)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:55:40.218395Z","iopub.execute_input":"2023-06-11T09:55:40.218755Z","iopub.status.idle":"2023-06-11T09:55:40.22654Z","shell.execute_reply.started":"2023-06-11T09:55:40.218724Z","shell.execute_reply":"2023-06-11T09:55:40.225466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32\n\ntable = tf.lookup.StaticHashTable(\n    initializer=tf.lookup.KeyValueTensorInitializer(\n        keys=list(char_to_num.keys()),\n        values=list(char_to_num.values()),\n    ),\n    default_value=tf.constant(-1),\n    name=\"class_weight\"\n)\n\ndef decode_fn(record_bytes):\n    schema = {COL: tf.io.FixedLenFeature([FRAME_LEN], dtype=tf.float32) for COL in SEL_COLS}\n    schema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n    features = tf.io.parse_single_example(record_bytes, schema)\n    phrase = features[\"phrase\"]\n    landmarks = tf.convert_to_tensor([features[COL] for COL in SEL_COLS])\n    landmarks = tf.transpose(landmarks)\n    mask = tf.math.less(landmarks, -2)\n#     nan_tensor = tf.fill(tf.shape(landmarks), tf.constant(np.nan, dtype=tf.float32))\n    nan_tensor = tf.fill(tf.shape(landmarks), tf.constant(0, dtype=tf.float32))\n    landmarks = tf.where(mask, nan_tensor, landmarks)\n    \n    phrase = '#' + phrase + '$'\n    phrase = tf.strings.bytes_split(phrase)\n    phrase = table.lookup(phrase)\n    phrase = tf.pad(phrase, paddings=[[0, 64 - tf.shape(phrase)[0]]])\n    \n    return landmarks, phrase\n\ninpdir = \"/kaggle/input/aslfr-preprocess-dataset\"\ntffiles = df.file_id.map(lambda x: f'{inpdir}/tfds/{x}.tfrecord').unique()","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:55:41.64245Z","iopub.execute_input":"2023-06-11T09:55:41.642797Z","iopub.status.idle":"2023-06-11T09:55:44.551099Z","shell.execute_reply.started":"2023-06-11T09:55:41.642768Z","shell.execute_reply":"2023-06-11T09:55:44.55013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32\nval_len = int(0.2 * len(tffiles))\ntrain_dataset = tf.data.TFRecordDataset(tffiles[val_len:]).map(decode_fn).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)\nval_dataset = tf.data.TFRecordDataset(tffiles[:val_len]).map(decode_fn).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:55:44.556147Z","iopub.execute_input":"2023-06-11T09:55:44.558348Z","iopub.status.idle":"2023-06-11T09:55:45.551663Z","shell.execute_reply.started":"2023-06-11T09:55:44.558314Z","shell.execute_reply":"2023-06-11T09:55:45.550659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TokenEmbedding(layers.Layer):\n    def __init__(self, num_vocab=1000, maxlen=100, num_hid=64):\n        super().__init__()\n        self.emb = tf.keras.layers.Embedding(num_vocab, num_hid)\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=num_hid)\n\n    def call(self, x):\n        tf.print(\"x\",x[0].shape)\n        maxlen = tf.shape(x)[-1]\n        x = self.emb(x)\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)       \n        tf.print(\"positions\",positions[0].shape)\n\n        return x + positions\n\n\nclass LandmarkEmbedding(layers.Layer):\n    def __init__(self, num_hid=64, maxlen=100):\n        super().__init__()\n        self.conv1 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n        self.conv2 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n        self.conv3 = tf.keras.layers.Conv1D(\n            num_hid, 11, padding=\"same\", activation=\"relu\"\n        )\n#         self.lstm1 = tf.keras.layers.LSTM(num_hid, return_sequences=True, activation=\"relu\")\n#         self.lstm2 = tf.keras.layers.LSTM(num_hid, return_sequences=True, activation=\"relu\")\n#         self.lstm3 = tf.keras.layers.LSTM(num_hid, return_sequences=True, activation=\"relu\")\n#         self.dense1 = tf.keras.layers.Dense(num_hid, activation=\"relu\")\n#         self.dense2 = tf.keras.layers.Dense(num_hid, activation=\"relu\")\n#         self.dense3 = tf.keras.layers.Dense(num_hid, activation=\"relu\")\n\n    def call(self, x):\n        #x = x[..., tf.newaxis]\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n#         x = self.dense1(x)\n#         x = self.dense2(x)\n#         x = self.dense3(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-11T10:28:24.802808Z","iopub.execute_input":"2023-06-11T10:28:24.803403Z","iopub.status.idle":"2023-06-11T10:28:24.821707Z","shell.execute_reply.started":"2023-06-11T10:28:24.803363Z","shell.execute_reply":"2023-06-11T10:28:24.820656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerEncoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, rate=0.1):\n        super().__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dense(embed_dim),\n            ]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:56:44.126943Z","iopub.execute_input":"2023-06-11T09:56:44.127308Z","iopub.status.idle":"2023-06-11T09:56:44.136499Z","shell.execute_reply.started":"2023-06-11T09:56:44.127277Z","shell.execute_reply":"2023-06-11T09:56:44.135175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerDecoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, dropout_rate=0.1):\n        super().__init__()\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = layers.LayerNormalization(epsilon=1e-6)\n        self.self_att = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=embed_dim\n        )\n        self.enc_att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.self_dropout = layers.Dropout(0.5)\n        self.enc_dropout = layers.Dropout(0.1)\n        self.ffn_dropout = layers.Dropout(0.1)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dense(embed_dim),\n            ]\n        )\n\n    def causal_attention_mask(self, batch_size, n_dest, n_src, dtype):\n        \"\"\"Masks the upper half of the dot product matrix in self attention.\n\n        This prevents flow of information from future tokens to current token.\n        1's in the lower triangle, counting from the lower right corner.\n        \"\"\"\n        i = tf.range(n_dest)[:, None]\n        j = tf.range(n_src)\n        m = i >= j - n_src + n_dest\n        mask = tf.cast(m, dtype)\n        mask = tf.reshape(mask, [1, n_dest, n_src])\n        mult = tf.concat(\n            #[tf.expand_dims(batch_size, -1), tf.constant([1, 1], dtype=tf.int32)], 0\n            [batch_size[..., tf.newaxis], tf.constant([1, 1], dtype=tf.int32)], 0\n        )\n        return tf.tile(mask, mult)\n\n    def call(self, enc_out, target):\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        seq_len = input_shape[1]\n        causal_mask = self.causal_attention_mask(batch_size, seq_len, seq_len, tf.bool)\n        target_att = self.self_att(target, target, attention_mask=causal_mask)\n        target_norm = self.layernorm1(target + self.self_dropout(target_att))\n        enc_out = self.enc_att(target_norm, enc_out)\n        enc_out_norm = self.layernorm2(self.enc_dropout(enc_out) + target_norm)\n        ffn_out = self.ffn(enc_out_norm)\n        ffn_out_norm = self.layernorm3(enc_out_norm + self.ffn_dropout(ffn_out))\n        return ffn_out_norm","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:55:52.157055Z","iopub.execute_input":"2023-06-11T09:55:52.157948Z","iopub.status.idle":"2023-06-11T09:55:52.172722Z","shell.execute_reply.started":"2023-06-11T09:55:52.157902Z","shell.execute_reply":"2023-06-11T09:55:52.171588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nclass Transformer(keras.Model):\n    def __init__(\n        self,\n        num_hid=64,\n        num_head=2,\n        num_feed_forward=128,\n        source_maxlen=100,\n        target_maxlen=100,\n        num_layers_enc=4,\n        num_layers_dec=1,\n        num_classes=10,\n    ):\n        super().__init__()\n        self.loss_metric = keras.metrics.Mean(name=\"loss\")\n        self.num_layers_enc = num_layers_enc\n        self.num_layers_dec = num_layers_dec\n        self.target_maxlen = target_maxlen\n        self.num_classes = num_classes\n\n        self.enc_input = LandmarkEmbedding(num_hid=num_hid, maxlen=source_maxlen)\n        self.encoder = keras.Sequential(\n            [self.enc_input]\n            + [\n                TransformerEncoder(num_hid, num_head, num_feed_forward)\n                for _ in range(num_layers_enc)\n            ]\n        )\n        self.dec_input = TokenEmbedding(\n            num_vocab=num_classes, maxlen=target_maxlen, num_hid=num_hid\n        )\n        for i in range(num_layers_dec):\n            setattr(\n                self,\n                f\"dec_layer_{i}\",\n                TransformerDecoder(num_hid, num_head, num_feed_forward),\n            )\n        self.classifier = layers.Dense(num_classes)\n\n    def decode(self, enc_out, target):\n        y = self.dec_input(target)\n        tf.print(\"decy\",y[0].shape,y[0])\n        for i in range(self.num_layers_dec):\n            y = getattr(self, f\"dec_layer_{i}\")(enc_out, y)\n        tf.print(\"decy\",y[0].shape,y[0])  \n        return y\n\n    def call(self, inputs):\n        source = inputs[0]\n        target = inputs[1]\n        tf.print(\"source\",source[0].shape)\n        tf.print(\"target\",target[0].shape)\n        x = self.encoder(source)\n        tf.print(\"x\",x[0].shape)\n        y = self.decode(x, target)\n        #tf.print(\"y\",y.shape)\n        tf.print(\"y\",y[0].shape)\n        ans = self.classifier(y)\n        tf.print(\"ans\",ans[0].shape)\n        return ans\n\n    @property\n    def metrics(self):\n        return [self.loss_metric]\n\n    def train_step(self, batch):\n        \"\"\"Processes one batch inside model.fit().\"\"\"\n        source = batch[0]\n        target = batch[1]\n\n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        \n        '''    \n        dec_input = target[:, 1:][:, :-1]\n        dec_target = target[:, 1:][:, :-1]\n        '''\n        with tf.GradientTape() as tape:\n            preds = self([source, dec_input])\n            one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n            mask = tf.math.logical_not(tf.math.equal(dec_target, 0))\n            loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        trainable_vars = self.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n        self.loss_metric.update_state(loss)\n        return {\"loss\": self.loss_metric.result()}\n\n    def test_step(self, batch):\n        source = batch[0]\n        target = batch[1]\n        \n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        '''    \n        dec_input = target[:, 1:][:, :-1]\n        dec_target = target[:, 1:][:, :-1]\n        '''\n        preds = self([source, dec_input])\n        one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n        mask = tf.math.logical_not(tf.math.equal(dec_target, 0))\n        loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        self.loss_metric.update_state(loss)\n        return {\"loss\": self.loss_metric.result()}\n\n    def generate(self, source, target_start_token_idx):\n        \"\"\"Performs inference over one batch of inputs using greedy decoding.\"\"\"\n        bs = tf.shape(source)[0]\n        enc = self.encoder(source)\n        dec_input = tf.ones((bs, 1), dtype=tf.int32) * target_start_token_idx\n        dec_logits = []\n        for i in range(self.target_maxlen - 1):\n            dec_out = self.decode(enc, dec_input)\n            logits = self.classifier(dec_out)\n            logits = tf.argmax(logits, axis=-1, output_type=tf.int32)\n            #last_logit = tf.expand_dims(logits[:, -1], axis=-1)\n            last_logit = logits[:, -1][..., tf.newaxis]\n            dec_logits.append(last_logit)\n            dec_input = tf.concat([dec_input, last_logit], axis=-1)\n        return dec_input","metadata":{"execution":{"iopub.status.busy":"2023-06-11T10:39:47.115462Z","iopub.execute_input":"2023-06-11T10:39:47.116074Z","iopub.status.idle":"2023-06-11T10:39:47.147999Z","shell.execute_reply.started":"2023-06-11T10:39:47.116034Z","shell.execute_reply":"2023-06-11T10:39:47.147035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DisplayOutputs(keras.callbacks.Callback):\n    def __init__(\n        self, batch, idx_to_token, target_start_token_idx=27, target_end_token_idx=28\n    ):\n        \"\"\"Displays a batch of outputs after every epoch\n\n        Args:\n            batch: A test batch containing the keys \"source\" and \"target\"\n            idx_to_token: A List containing the vocabulary tokens corresponding to their indices\n            target_start_token_idx: A start token index in the target vocabulary\n            target_end_token_idx: An end token index in the target vocabulary\n        \"\"\"\n        self.batch = batch\n        self.target_start_token_idx = target_start_token_idx\n        self.target_end_token_idx = target_end_token_idx\n        self.idx_to_char = idx_to_token\n\n    def on_epoch_end(self, epoch, logs=None):\n#         if epoch % 5 != 0:\n#             return\n        source = self.batch[0]\n        target = self.batch[1].numpy()\n        bs = tf.shape(source)[0]\n        preds = self.model.generate(source, self.target_start_token_idx)\n        preds = preds.numpy()\n        for i in range(bs):\n            target_text = \"\".join([self.idx_to_char[_] for _ in target[i, :]])\n            prediction = \"\"\n            for idx in preds[i, :]:\n                prediction += self.idx_to_char[idx]\n                if idx == self.target_end_token_idx:\n                    break\n            print(f\"target:     {target_text.replace('-','')}\")\n            print(f\"prediction: {prediction}\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-06-11T10:28:42.615505Z","iopub.execute_input":"2023-06-11T10:28:42.615855Z","iopub.status.idle":"2023-06-11T10:28:42.625913Z","shell.execute_reply.started":"2023-06-11T10:28:42.615827Z","shell.execute_reply":"2023-06-11T10:28:42.624958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch = next(iter(val_dataset))\n\nidx_to_char = list(char_to_num.keys())\ndisplay_cb = DisplayOutputs(\n    batch, num_to_char, target_start_token_idx=2, target_end_token_idx=3\n)  # set the arguments as per vocabulary index for '<' and '>'\n\nmodel = Transformer(\n    num_hid=200,\n    num_head=2,\n    num_feed_forward=400,\n    target_maxlen=64,\n    num_layers_enc=4,\n    num_layers_dec=1,\n    num_classes=59,\n)\n\nloss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.1,)\noptimizer = keras.optimizers.Adam(0.0001)\nmodel.compile(optimizer=optimizer, loss=loss_fn)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T10:39:53.821126Z","iopub.execute_input":"2023-06-11T10:39:53.821492Z","iopub.status.idle":"2023-06-11T10:39:54.016371Z","shell.execute_reply.started":"2023-06-11T10:39:53.821461Z","shell.execute_reply":"2023-06-11T10:39:54.015351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_dataset, validation_data=val_dataset, callbacks=[display_cb], epochs=10)\n# history = model.fit(train_dataset.take(1), validation_data=val_dataset.take(1), callbacks=[display_cb], epochs=1)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T10:39:55.358036Z","iopub.execute_input":"2023-06-11T10:39:55.35842Z","iopub.status.idle":"2023-06-11T10:40:17.854353Z","shell.execute_reply.started":"2023-06-11T10:39:55.358391Z","shell.execute_reply":"2023-06-11T10:40:17.851876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\n# Define two embeddings\nembedding1 = tf.Variable(tf.random.normal([10, 10, 5]))  # Shape: (batch_size, embedding_dim)\nembedding2 = tf.Variable(tf.random.normal([10, 5])*100)  # Shape: (batch_size, embedding_dim)\n\n# Add the embeddings\nsum_embedding = embedding1 + embedding2\n\n# Print the resulting sum embedding\nprint(embedding1)\nprint(sum_embedding)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T08:41:34.398643Z","iopub.execute_input":"2023-06-11T08:41:34.399072Z","iopub.status.idle":"2023-06-11T08:41:34.432023Z","shell.execute_reply.started":"2023-06-11T08:41:34.399037Z","shell.execute_reply":"2023-06-11T08:41:34.43097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T13:53:44.443713Z","iopub.execute_input":"2023-06-07T13:53:44.444095Z","iopub.status.idle":"2023-06-07T13:53:44.485953Z","shell.execute_reply.started":"2023-06-07T13:53:44.444045Z","shell.execute_reply":"2023-06-07T13:53:44.485037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n        \n    def __call__(self, x):\n        #x = tf.expand_dims(x, 0)\n        x = x[None]\n        x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n        x = tf.image.resize(x, (tf.shape(x)[0], FRAME_LEN))\n        x = x[0]\n        return x\n    \nclass TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n\n        # Load the feature generation and main models\n        self.preprocess_layer = PreprocessLayer()\n        self.model = model\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(SEL_COLS)], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs, training=False):\n        # Preprocess Data\n        x = self.preprocess_layer(inputs)\n        #x = tf.expand_dims(x, 0)\n        x = x[None]\n        x = self.model.generate(x, 2)\n        x = x[0]\n        idx = tf.argmax(tf.cast(tf.equal(x, 3), tf.int32))\n        idx = tf.where(tf.math.less(idx, 1), tf.constant(3, dtype=tf.int64), idx)\n        x = x[1:idx]\n        x = tf.one_hot(x, 59)\n        return {'outputs': x}\n    \npre = PreprocessLayer()\nprint(pre(batch[0][0]).shape)\ntflitemodel_base = TFLiteModel(model)\nprint(batch[0][0].shape)\ntflitemodel_base(batch[0][0])[\"outputs\"].shape","metadata":{"execution":{"iopub.status.busy":"2023-05-31T08:10:57.728108Z","iopub.execute_input":"2023-05-31T08:10:57.728528Z","iopub.status.idle":"2023-05-31T08:11:14.374276Z","shell.execute_reply.started":"2023-05-31T08:10:57.728496Z","shell.execute_reply":"2023-05-31T08:11:14.373028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights(\"model.h5\")","metadata":{"execution":{"iopub.status.busy":"2023-05-31T08:12:49.471505Z","iopub.execute_input":"2023-05-31T08:12:49.471921Z","iopub.status.idle":"2023-05-31T08:12:49.605332Z","shell.execute_reply.started":"2023-05-31T08:12:49.471891Z","shell.execute_reply":"2023-05-31T08:12:49.604296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.load_weights(\"model.h5\")","metadata":{"execution":{"iopub.status.busy":"2023-05-31T08:09:58.893861Z","iopub.execute_input":"2023-05-31T08:09:58.894285Z","iopub.status.idle":"2023-05-31T08:09:59.568622Z","shell.execute_reply.started":"2023-05-31T08:09:58.894252Z","shell.execute_reply":"2023-05-31T08:09:59.567412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]#, tf.lite.OpsSet.SELECT_TF_OPS]\ntflite_model = keras_model_converter.convert()\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \ninfargs = {\"selected_columns\" : SEL_COLS}\n\nwith open('inference_args.json', \"w\") as json_file:\n    json.dump(infargs, json_file)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T08:11:18.029424Z","iopub.execute_input":"2023-05-31T08:11:18.029885Z","iopub.status.idle":"2023-05-31T08:12:28.692918Z","shell.execute_reply.started":"2023-05-31T08:11:18.02985Z","shell.execute_reply":"2023-05-31T08:12:28.691851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip  './model.tflite' './inference_args.json'","metadata":{"execution":{"iopub.status.busy":"2023-05-31T08:12:32.744929Z","iopub.execute_input":"2023-05-31T08:12:32.745369Z","iopub.status.idle":"2023-05-31T08:12:35.149079Z","shell.execute_reply.started":"2023-05-31T08:12:32.74534Z","shell.execute_reply":"2023-05-31T08:12:35.147518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tf.lite.Interpreter(\"model.tflite\")\n\nREQUIRED_SIGNATURE = \"serving_default\"\nREQUIRED_OUTPUT = \"outputs\"\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\nif REQUIRED_SIGNATURE not in found_signatures:\n    raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=batch[0][0])\nprediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\nprint(prediction_str)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T08:12:56.059267Z","iopub.execute_input":"2023-05-31T08:12:56.05967Z","iopub.status.idle":"2023-05-31T08:12:56.32032Z","shell.execute_reply.started":"2023-05-31T08:12:56.059642Z","shell.execute_reply":"2023-05-31T08:12:56.319041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape = (32, 128, 342)\nx = tf.random.normal(input_shape)\nconv1D = tf.keras.layers.Conv1D(356, 4, padding='valid')  #1是输出通道数，3是卷积核大小，不使用边界填充\nmax_pool_1d = tf.keras.layers.MaxPooling1D(pool_size=2,strides=1, padding='valid')\ny= conv1D(x)\ny = max_pool_1d(y)\nprint(y)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T18:13:42.640558Z","iopub.execute_input":"2023-06-07T18:13:42.64094Z","iopub.status.idle":"2023-06-07T18:13:42.691956Z","shell.execute_reply.started":"2023-06-07T18:13:42.640909Z","shell.execute_reply":"2023-06-07T18:13:42.690928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport tensorflow as tf\nimport numpy as np\n\nREGULARIZER = tf.keras.regularizers.l2(l=0.0001)\nINITIALIZER = tf.keras.initializers.VarianceScaling(scale=2.,\n                                                    mode=\"fan_out\",\n                                                    distribution=\"truncated_normal\")\n\n\n\"\"\"The basic module for applying a spatial graph convolution.\n    Args:\n        filters (int): Number of channels produced by the convolution\n        kernel_size (int): Size of the graph convolving kernel\n    Shape:\n        - Input[0]: Input graph sequence in :math:`(N, C, T, V)` format\n        - Input[1]: Input graph adjacency matrix in :math:`(K, V, V)` format\n        - Output[0]: Output graph sequence in :math:`(N, out_channels, T, V)` format\n        - Output[1]: Graph adjacency matrix for output data in :math:`(K, V, V)` format\n        where\n            :math:`N` is a batch size\n            :math:`K` is the spatial kernel size\n            :math:`T` is a length of the sequence\n            :math:`V` is the number of graph nodes\n            :math:`C` is the number of incoming channels\n\"\"\"\nclass SGCN(tf.keras.Model):\n    def __init__(self, filters, kernel_size=3):\n        super().__init__()\n        self.kernel_size = kernel_size\n        self.conv = tf.keras.layers.Conv2D(filters*kernel_size,\n                                           kernel_size=1,\n                                           padding='same',\n                                           kernel_initializer=INITIALIZER,\n                                           data_format='channels_first',\n                                           kernel_regularizer=REGULARIZER)\n\n    # N, C, T, V\n    def call(self, x, A, training):\n        x = self.conv(x)\n\n        N = tf.shape(x)[0]\n        C = tf.shape(x)[1]\n        T = tf.shape(x)[2]\n        V = tf.shape(x)[3]\n\n        x = tf.reshape(x, [N, self.kernel_size, C//self.kernel_size, T, V])\n        x = tf.einsum('nkctv,kvw->nctw', x, A)\n        return x, A\n\n\n\"\"\"Applies a spatial temporal graph convolution over an input graph sequence.\n    Args:\n        filters (int): Number of channels produced by the convolution\n        kernel_size (tuple): Size of the temporal convolving kernel and graph convolving kernel\n        stride (int, optional): Stride of the temporal convolution. Default: 1\n        activation (activation function/name, optional): activation function to use\n        residual (bool, optional): If ``True``, applies a residual mechanism. Default: ``True``\n        downsample (bool, optional): If ``True``, applies a downsampling residual mechanism. Default: ``True``\n                                     the value is used only when residual is ``True``\n    Shape:\n        - Input[0]: Input graph sequence in :math:`(N, in_channels, T_{in}, V)` format\n        - Input[1]: Input graph adjacency matrix in :math:`(K, V, V)` format\n        - Output[0]: Outpu graph sequence in :math:`(N, out_channels, T_{out}, V)` format\n        - Output[1]: Graph adjacency matrix for output data in :math:`(K, V, V)` format\n        where\n            :math:`N` is a batch size,\n            :math:`K` is the spatial kernel size, as :math:`K == kernel_size[1]`,\n            :math:`T_{in}/T_{out}` is a length of input/output sequence,\n            :math:`V` is the number of graph nodes.\n\"\"\"\nclass STGCN(tf.keras.Model):\n    def __init__(self, filters, kernel_size=[9, 3], stride=1, activation='relu',\n                 residual=True, downsample=False):\n        super().__init__()\n        self.sgcn = SGCN(filters, kernel_size=kernel_size[1])\n\n        self.tgcn = tf.keras.Sequential()\n        self.tgcn.add(tf.keras.layers.BatchNormalization(axis=1))\n        self.tgcn.add(tf.keras.layers.Activation(activation))\n        self.tgcn.add(tf.keras.layers.Conv2D(filters,\n                                                kernel_size=[kernel_size[0], 1],\n                                                strides=[stride, 1],\n                                                padding='same',\n                                                kernel_initializer=INITIALIZER,\n                                                data_format='channels_first',\n                                                kernel_regularizer=REGULARIZER))\n        self.tgcn.add(tf.keras.layers.BatchNormalization(axis=1))\n\n        self.act = tf.keras.layers.Activation(activation)\n\n        if not residual:\n            self.residual = lambda x, training=False: 0\n        elif residual and stride == 1 and not downsample:\n            self.residual = lambda x, training=False: x\n        else:\n            self.residual = tf.keras.Sequential()\n            self.residual.add(tf.keras.layers.Conv2D(filters,\n                                                        kernel_size=[1, 1],\n                                                        strides=[stride, 1],\n                                                        padding='same',\n                                                        kernel_initializer=INITIALIZER,\n                                                        data_format='channels_first',\n                                                        kernel_regularizer=REGULARIZER))\n            self.residual.add(tf.keras.layers.BatchNormalization(axis=1))\n\n    def call(self, x, A, training):\n        res = self.residual(x, training=training)\n        x, A = self.sgcn(x, A, training=training)\n        x = self.tgcn(x, training=training)\n        x += res\n        x = self.act(x)\n        return x, A\n\n\n\"\"\"Spatial temporal graph convolutional networks.\n    Args:\n        num_class (int): Number of classes for the classification task\n    Shape:\n        - Input: :math:`(N, in_channels, T_{in}, V_{in}, M_{in})`\n        - Output: :math:`(N, num_class)` where\n            :math:`N` is a batch size,\n            :math:`T_{in}` is a length of input sequence,\n            :math:`V_{in}` is the number of graph nodes,\n            :math:`M_{in}` is the number of instance in a frame.\n\"\"\"\nclass Model(tf.keras.Model):\n    def __init__(self, num_classes=60):\n        super().__init__()\n\n        graph = Graph()\n        self.A = tf.Variable(graph.A,\n                             dtype=tf.float32,\n                             trainable=False,\n                             name='adjacency_matrix')\n\n        self.data_bn = tf.keras.layers.BatchNormalization(axis=1)\n\n        self.STGCN_layers = []\n        self.STGCN_layers.append(STGCN(64, residual=False))\n        self.STGCN_layers.append(STGCN(64))\n        self.STGCN_layers.append(STGCN(64))\n        self.STGCN_layers.append(STGCN(64))\n        self.STGCN_layers.append(STGCN(128, stride=2, downsample=True))\n        self.STGCN_layers.append(STGCN(128))\n        self.STGCN_layers.append(STGCN(128))\n        self.STGCN_layers.append(STGCN(256, stride=2, downsample=True))\n        self.STGCN_layers.append(STGCN(256))\n        self.STGCN_layers.append(STGCN(256))\n\n        self.pool = tf.keras.layers.GlobalAveragePooling2D(data_format='channels_first')\n\n        self.logits = tf.keras.layers.Conv2D(num_classes,\n                                             kernel_size=1,\n                                             padding='same',\n                                             kernel_initializer=INITIALIZER,\n                                             data_format='channels_first',\n                                             kernel_regularizer=REGULARIZER)\n\n    def call(self, x, training):\n        N = tf.shape(x)[0]\n        C = tf.shape(x)[1]\n        T = tf.shape(x)[2]\n        V = tf.shape(x)[3]\n        M = tf.shape(x)[4]\n\n        x = tf.transpose(x, perm=[0, 4, 3, 1, 2])\n        x = tf.reshape(x, [N * M, V * C, T])\n        x = self.data_bn(x, training=training)\n        x = tf.reshape(x, [N, M, V, C, T])\n        x = tf.transpose(x, perm=[0, 1, 3, 4, 2])\n        x = tf.reshape(x, [N * M, C, T, V])\n\n        A = self.A\n        for layer in self.STGCN_layers:\n            x, A = layer(x, A, training=training)\n\n        # N*M,C,T,V\n        x = self.pool(x)\n        x = tf.reshape(x, [N, M, -1, 1, 1])\n        x = tf.reduce_mean(x, axis=1)\n        x = self.logits(x)\n        x = tf.reshape(x, [N, -1])\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:09:50.156824Z","iopub.execute_input":"2023-06-08T06:09:50.157192Z","iopub.status.idle":"2023-06-08T06:09:57.707428Z","shell.execute_reply.started":"2023-06-08T06:09:50.157162Z","shell.execute_reply":"2023-06-08T06:09:57.705254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n\ndef edge2mat(link, num_node):\n    A = np.zeros((num_node, num_node))\n    for i, j in link:\n        A[j, i] = 1\n    return A\n\n\ndef normalize_digraph(A):  # 除以每列的和\n    Dl = np.sum(A, 0)\n    h, w = A.shape\n    Dn = np.zeros((w, w))\n    for i in range(w):\n        if Dl[i] > 0:\n            Dn[i, i] = Dl[i] ** (-1)\n    AD = np.dot(A, Dn)\n    return AD\n\n\ndef get_spatial_graph(num_node, self_link, inward, outward):\n    I = edge2mat(self_link, num_node)\n    In = normalize_digraph(edge2mat(inward, num_node))\n    Out = normalize_digraph(edge2mat(outward, num_node))\n    A = np.stack((I, In, Out))\n    return A","metadata":{"execution":{"iopub.status.busy":"2023-06-08T14:26:23.234843Z","iopub.execute_input":"2023-06-08T14:26:23.235189Z","iopub.status.idle":"2023-06-08T14:26:23.244602Z","shell.execute_reply.started":"2023-06-08T14:26:23.235161Z","shell.execute_reply":"2023-06-08T14:26:23.243672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\nnum_node = 25\nself_link = [(i, i) for i in range(num_node)]\ninward_ori_index = [(1, 2), (2, 21), (3, 21), (4, 3), (5, 21), (6, 5), (7, 6),\n                    (8, 7), (9, 21), (10, 9), (11, 10), (12, 11), (13, 1),\n                    (14, 13), (15, 14), (16, 15), (17, 1), (18, 17), (19, 18),\n                    (20, 19), (22, 23), (23, 8), (24, 25), (25, 12)]\ninward = [(i - 1, j - 1) for (i, j) in inward_ori_index]\noutward = [(j, i) for (i, j) in inward]\nneighbor = inward + outward\n\n\nclass Graph:\n    def __init__(self, labeling_mode='spatial'):\n        self.A = self.get_adjacency_matrix(labeling_mode)\n        self.num_node = num_node\n        self.self_link = self_link\n        self.inward = inward\n        self.outward = outward\n        self.neighbor = neighbor\n\n    def get_adjacency_matrix(self, labeling_mode=None):\n        if labeling_mode is None:\n            return self.A\n        if labeling_mode == 'spatial':\n            A = get_spatial_graph(num_node, self_link, inward, outward)\n        else:\n            raise ValueError()\n        return A\n\n\nif __name__ == '__main__':\n    import matplotlib.pyplot as plt\n    import os\n\n    # os.environ['DISPLAY'] = 'localhost:11.0'\n    A = Graph('spatial').get_adjacency_matrix()\n    for i in A:\n        plt.imshow(i, cmap='gray')\n        plt.show()\n    print(A)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T14:26:25.699227Z","iopub.execute_input":"2023-06-08T14:26:25.699628Z","iopub.status.idle":"2023-06-08T14:26:26.38568Z","shell.execute_reply.started":"2023-06-08T14:26:25.699591Z","shell.execute_reply":"2023-06-08T14:26:26.384714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport tensorflow as tf\nimport numpy as np\n\nREGULARIZER = tf.keras.regularizers.l2(l=0.0001)\nINITIALIZER = tf.keras.initializers.VarianceScaling(scale=2.,\n                                                    mode=\"fan_out\",\n                                                    distribution=\"truncated_normal\")\nclass SGCN(tf.keras.Model):\n    def __init__(self, filters, kernel_size=3):\n        super().__init__()\n        self.kernel_size = kernel_size\n        self.conv = tf.keras.layers.Conv2D(filters*kernel_size,\n                                           kernel_size=1,\n                                           padding='same',\n                                           kernel_initializer=INITIALIZER,\n                                           data_format='channels_first',\n                                           kernel_regularizer=REGULARIZER)\n\n    # N, C, T, V\n    def call(self, x, A, training):\n        x = self.conv(x)\n\n        N = tf.shape(x)[0]\n        C = tf.shape(x)[1]\n        T = tf.shape(x)[2]\n        V = tf.shape(x)[3]\n\n        x = tf.reshape(x, [N, self.kernel_size, C//self.kernel_size, T, V])\n        x = tf.einsum('nkctv,kvw->nctw', x, A)\n        return x, A\n    \nclass STGCN(tf.keras.Model):\n    def __init__(self, filters, kernel_size=[9, 3], stride=1, activation='relu',\n                 residual=True, downsample=False):\n        super().__init__()\n        self.sgcn = SGCN(filters, kernel_size=kernel_size[1])\n\n        self.tgcn = tf.keras.Sequential()\n        self.tgcn.add(tf.keras.layers.BatchNormalization(axis=1))\n        self.tgcn.add(tf.keras.layers.Activation(activation))\n        self.tgcn.add(tf.keras.layers.Conv2D(filters,\n                                                kernel_size=[kernel_size[0], 1],\n                                                strides=[stride, 1],\n                                                padding='same',\n                                                kernel_initializer=INITIALIZER,\n                                                data_format='channels_first',\n                                                kernel_regularizer=REGULARIZER))\n        self.tgcn.add(tf.keras.layers.BatchNormalization(axis=1))\n\n        self.act = tf.keras.layers.Activation(activation)\n\n        if not residual:\n            self.residual = lambda x, training=False: 0\n        elif residual and stride == 1 and not downsample:\n            self.residual = lambda x, training=False: x\n        else:\n            self.residual = tf.keras.Sequential()\n            self.residual.add(tf.keras.layers.Conv2D(filters,\n                                                        kernel_size=[1, 1],\n                                                        strides=[stride, 1],\n                                                        padding='same',\n                                                        kernel_initializer=INITIALIZER,\n                                                        data_format='channels_first',\n                                                        kernel_regularizer=REGULARIZER))\n            self.residual.add(tf.keras.layers.BatchNormalization(axis=1))\n\n    def call(self, x, A, training):\n        res = self.residual(x, training=training)\n        x, A = self.sgcn(x, A, training=training)\n        x = self.tgcn(x, training=training)\n        x += res\n        x = self.act(x)\n        return x, A","metadata":{"execution":{"iopub.status.busy":"2023-06-08T14:26:29.482412Z","iopub.execute_input":"2023-06-08T14:26:29.482819Z","iopub.status.idle":"2023-06-08T14:26:29.502594Z","shell.execute_reply.started":"2023-06-08T14:26:29.482772Z","shell.execute_reply":"2023-06-08T14:26:29.501538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(tf.keras.Model):\n    def __init__(self, num_classes=60):\n        super().__init__()\n\n        graph = Graph()\n        self.A = tf.Variable(graph.A,\n                             dtype=tf.float32,\n                             trainable=False,\n                             name='adjacency_matrix')\n\n        self.data_bn = tf.keras.layers.BatchNormalization(axis=1)\n\n        self.STGCN_layers = []\n        self.STGCN_layers.append(STGCN(64, residual=False))\n        self.STGCN_layers.append(STGCN(64))\n        self.STGCN_layers.append(STGCN(64))\n        self.STGCN_layers.append(STGCN(64))\n        self.STGCN_layers.append(STGCN(128, stride=2, downsample=True))\n        self.STGCN_layers.append(STGCN(128))\n        self.STGCN_layers.append(STGCN(128))\n        self.STGCN_layers.append(STGCN(256, stride=2, downsample=True))\n        self.STGCN_layers.append(STGCN(256))\n        self.STGCN_layers.append(STGCN(256))\n\n        self.pool = tf.keras.layers.GlobalAveragePooling2D(data_format='channels_first')\n\n        self.logits = tf.keras.layers.Conv2D(num_classes,\n                                             kernel_size=1,\n                                             padding='same',\n                                             kernel_initializer=INITIALIZER,\n                                             data_format='channels_first',\n                                             kernel_regularizer=REGULARIZER)\n\n    def call(self, x, training):\n        N = tf.shape(x)[0]\n        C = tf.shape(x)[1]\n        T = tf.shape(x)[2]\n        V = tf.shape(x)[3]\n        M = tf.shape(x)[4]\n\n        x = tf.transpose(x, perm=[0, 4, 3, 1, 2])\n        x = tf.reshape(x, [N * M, V * C, T])\n        x = self.data_bn(x, training=training)\n        x = tf.reshape(x, [N, M, V, C, T])\n        x = tf.transpose(x, perm=[0, 1, 3, 4, 2])\n        x = tf.reshape(x, [N * M, C, T, V])\n\n        A = self.A\n        for layer in self.STGCN_layers:\n            x, A = layer(x, A, training=training)\n\n        # N*M,C,T,V\n        x = self.pool(x)\n        x = tf.reshape(x, [N, M, -1, 1, 1])\n        x = tf.reduce_mean(x, axis=1)\n        x = self.logits(x)\n        x = tf.reshape(x, [N, -1])\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-08T14:26:32.607662Z","iopub.execute_input":"2023-06-08T14:26:32.608328Z","iopub.status.idle":"2023-06-08T14:26:32.624286Z","shell.execute_reply.started":"2023-06-08T14:26:32.608294Z","shell.execute_reply":"2023-06-08T14:26:32.623303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model()\nmodel.build(input_shape=(32, 3, 128, 25, 1))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:58:42.682532Z","iopub.execute_input":"2023-06-08T06:58:42.68306Z","iopub.status.idle":"2023-06-08T06:58:44.31974Z","shell.execute_reply.started":"2023-06-08T06:58:42.683027Z","shell.execute_reply":"2023-06-08T06:58:44.318785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape = (32, 3, 128, 25, 1)\nx = tf.random.normal(input_shape)\nN = tf.shape(x)[0]\nC = tf.shape(x)[1]\nT = tf.shape(x)[2]\nV = tf.shape(x)[3]\nM = tf.shape(x)[4]\n\nx = tf.transpose(x, perm=[0, 1, 3, 4, 2])\nx = tf.reshape(x, [N * M, C, T, V])\n\nlayer1 = STGCN(16,kernel_size=[9, 3], residual=False)\nlayer2 = STGCN(32, stride=2, downsample=True)\nlayer3 = STGCN(32)\nlayer4 = STGCN(64, stride=2, downsample=True)\nlayer5 = STGCN(64)\npool = tf.keras.layers.GlobalAveragePooling2D(data_format='channels_first')\nlogit = tf.keras.layers.Conv2D(10,\n                         kernel_size=1,\n                         padding='same',\n                         kernel_initializer=INITIALIZER,\n                         data_format='channels_first',\n                         kernel_regularizer=REGULARIZER)\nx = layer1(x,A)\nx = layer2(x[0],A)\nx = layer3(x[0],A)\nx = layer4(x[0],A)\nx = layer5(x[0],A)\nprint(x)\nx = pool(x[0])\nx = tf.reshape(x, [N, M, -1, 1, 1])\nx = tf.reduce_mean(x, axis=1)\nx = logit(x)\nx = tf.reshape(x, [N, -1])\n\nprint(x)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T14:38:08.309687Z","iopub.execute_input":"2023-06-08T14:38:08.310061Z","iopub.status.idle":"2023-06-08T14:38:08.854301Z","shell.execute_reply.started":"2023-06-08T14:38:08.310032Z","shell.execute_reply":"2023-06-08T14:38:08.853373Z"},"trusted":true},"execution_count":null,"outputs":[]}]}