{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pyarrow.parquet as pq\nimport tensorflow as tf\nimport keras\nfrom keras import layers\nimport pandas as pd\nimport numpy as np\nimport json\nimport os\nimport shutil\nimport tqdm\nfrom tensorflow.keras.utils import Progbar\nimport glob\nimport logging\nimport gc\nimport multiprocessing as mp\n","metadata":{"execution":{"iopub.status.busy":"2023-08-24T19:42:13.26222Z","iopub.execute_input":"2023-08-24T19:42:13.26258Z","iopub.status.idle":"2023-08-24T19:42:21.795259Z","shell.execute_reply.started":"2023-08-24T19:42:13.262549Z","shell.execute_reply":"2023-08-24T19:42:21.794279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tpu = None\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=\"local\") # \"local\" for 1VM TPU\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print(\"on TPU\")\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\nexcept:\n    strategy = tf.distribute.get_strategy()","metadata":{"execution":{"iopub.status.busy":"2023-08-24T19:42:21.800275Z","iopub.execute_input":"2023-08-24T19:42:21.803132Z","iopub.status.idle":"2023-08-24T19:42:24.614758Z","shell.execute_reply.started":"2023-08-24T19:42:21.803094Z","shell.execute_reply":"2023-08-24T19:42:24.613752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# resolver = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n# # tf.config.experimental_connect_to_cluster(resolver)\n# # tf.tpu.experimental.initialize_tpu_system(resolver)\n# print(\"All devices: \", tf.config.list_logical_devices(\"TPU\"))","metadata":{"execution":{"iopub.status.busy":"2023-08-24T19:42:24.616124Z","iopub.execute_input":"2023-08-24T19:42:24.616468Z","iopub.status.idle":"2023-08-24T19:42:24.621548Z","shell.execute_reply.started":"2023-08-24T19:42:24.616436Z","shell.execute_reply":"2023-08-24T19:42:24.620549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logging.basicConfig(level=logging.DEBUG, filename=\"training_log.log\", encoding=\"utf-8\",\n                    filemode=\"w\", format=\"%(asctime)s - %(levelname)s - %(message)s\")\nlogging.debug(\"\\n\\n\")\n# print(\"Current working directory: {}\".format(os.getcwd()))\n\n# cwd = \"/Users/arvinprince/pytorch-files/Google-ASL\"\n# print(\"Setting the working directory to : {}\".format(cwd))\n# os.chdir(cwd)\ndataset_df = pd.read_csv(\"/kaggle/input/asl-fingerspelling/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-08-24T19:42:24.623678Z","iopub.execute_input":"2023-08-24T19:42:24.624569Z","iopub.status.idle":"2023-08-24T19:42:24.800915Z","shell.execute_reply.started":"2023-08-24T19:42:24.624536Z","shell.execute_reply":"2023-08-24T19:42:24.799858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FRAME_LEN = 128\n\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\n'''LIPS = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308]\nLIPS.sort()'''\n\nX = [f\"x_right_hand_{i}\" for i in range(21)] + [f\"x_left_hand_{i}\" for i in range(21)] + \\\n    [f\"x_pose_{i}\" for i in POSE]\\\n# + [f\"x_face_{i}\" for i in LIPS]\nY = [f\"y_right_hand_{i}\" for i in range(21)] + [f\"y_left_hand_{i}\" for i in range(21)] + \\\n    [f\"y_pose_{i}\" for i in POSE]\\\n# + [f\"y_face_{i}\" for i in LIPS]\nZ = [f\"z_right_hand_{i}\" for i in range(21)] + [f\"z_left_hand_{i}\" for i in range(21)] + \\\n    [f\"z_pose_{i}\" for i in POSE]\\\n# + [f\"z_face_{i}\" for i in LIPS]\n\n\nFEATURE_COLUMNS = X + Y + Z\n\nX_IDX = [i for i, col in enumerate(FEATURE_COLUMNS) if \"x_\" in col]\nY_IDX = [i for i, col in enumerate(FEATURE_COLUMNS) if \"y_\" in col]\nZ_IDX = [i for i, col in enumerate(FEATURE_COLUMNS) if \"z_\" in col]\n\n\nRHAND_IDX = [i for i, col in enumerate(FEATURE_COLUMNS) if \"right\" in col]\nLHAND_IDX = [i for i, col in enumerate(FEATURE_COLUMNS) if \"left\" in col]\nRPOSE_IDX = [i for i, col in enumerate(FEATURE_COLUMNS) if \"pose\" in col and int(col[-2:]) in RPOSE]\nLPOSE_IDX = [i for i, col in enumerate(FEATURE_COLUMNS) if \"pose\" in col and int(col[-2:]) in LPOSE]\n# LIPS_IDX = [i for i, col in enumerate(FEATURE_COLUMNS) if \"face\" in col]","metadata":{"execution":{"iopub.status.busy":"2023-08-24T19:43:25.697504Z","iopub.execute_input":"2023-08-24T19:43:25.698201Z","iopub.status.idle":"2023-08-24T19:43:25.710405Z","shell.execute_reply.started":"2023-08-24T19:43:25.698166Z","shell.execute_reply":"2023-08-24T19:43:25.709261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocessed_dir():\n    if not os.path.isdir(\"/kaggle/working/preprocessed\"):\n        os.mkdir(\"/kaggle/working/preprocessed\")\n    else:\n        shutil.rmtree(\"/kaggle/working/preprocessed\")\n        os.mkdir(\"/kaggle/working/preprocessed\")\n# Loop through each file_id\n# @jit(target_backend=\"cuda\")\ndef processing_pq():\n#     bar = tqdm.tqdm(dataset_df[\"file_id\"].unique(), desc=\"Parsing parquet files to TFRecord files\")\n    bar = Progbar(len(dataset_df[\"file_id\"].unique()))\n    for i, file_id in enumerate(dataset_df[\"file_id\"].unique()):        \n        # Parquet file name\n        '''\n        pq_file = f\"/kaggle/input/asl-fingerspelling/train_landmarks/{file_id}.parquet\"\n        '''\n        # Filter train.csv and fetch entries only for the relevant file_id\n        file_df = dataset_df.loc[dataset_df[\"file_id\"] == file_id]\n        # Fetch the parquet file\n        parquet_df = pq.read_table(f\"/kaggle/input/asl-fingerspelling/train_landmarks/{str(file_id)}.parquet\",\n                                   columns=['sequence_id'] + FEATURE_COLUMNS).to_pandas()\n        # File name for the updated data\n        tf_file = f\"/kaggle/working/preprocessed/{file_id}.tfrecord\"\n        parquet_numpy = parquet_df.to_numpy()\n        # Initialize the pointer to write the output of\n        # each `for loop` below as a sequence into the file.\n        with tf.io.TFRecordWriter(tf_file) as file_writer:\n            # Loop through each sequence in file.\n            for seq_id, phrase in zip(file_df.sequence_id, file_df.phrase):\n                # Fetch sequence data\n                frames = parquet_numpy[parquet_df.index == seq_id]\n\n                # Calculate the number of NaN values in each hand landmark\n                r_nonan = np.sum(np.sum(np.isnan(frames[:, RHAND_IDX]), axis=1) == 0)\n                l_nonan = np.sum(np.sum(np.isnan(frames[:, LHAND_IDX]), axis=1) == 0)\n                no_nan = max(r_nonan, l_nonan)\n\n                if 2 * len(phrase) < no_nan:\n                    features = {FEATURE_COLUMNS[i]: tf.train.Feature(\n                        float_list=tf.train.FloatList(value=frames[:, i])) for i in range(len(FEATURE_COLUMNS))}\n                    features[\"phrase\"] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[bytes(phrase, 'utf-8')]))\n                    record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n                    file_writer.write(record_bytes)\n        bar.update(len(os.listdir(\"/kaggle/working/preprocessed\")))","metadata":{"_uuid":"2b12e7b4-4778-4809-9e31-e7081837eb6f","_cell_guid":"919df93a-78dd-4cf9-ab88-223761ec4e88","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-24T19:44:35.469757Z","iopub.execute_input":"2023-08-24T19:44:35.470263Z","iopub.status.idle":"2023-08-24T19:44:35.483236Z","shell.execute_reply.started":"2023-08-24T19:44:35.470227Z","shell.execute_reply":"2023-08-24T19:44:35.48206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    preprocessed_dir()\n    p = mp.Pool(processes=os.cpu_count())\n    p.apply_async(processing_pq)\n    \n    p.close()\n    p.join()","metadata":{"execution":{"iopub.status.busy":"2023-08-24T19:44:37.909905Z","iopub.execute_input":"2023-08-24T19:44:37.910477Z","iopub.status.idle":"2023-08-24T19:54:36.532137Z","shell.execute_reply.started":"2023-08-24T19:44:37.910445Z","shell.execute_reply":"2023-08-24T19:54:36.530717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tf.config.list_logical_devices(\"GPU\")","metadata":{"execution":{"iopub.status.busy":"2023-08-24T19:55:05.304519Z","iopub.execute_input":"2023-08-24T19:55:05.304908Z","iopub.status.idle":"2023-08-24T19:55:05.310395Z","shell.execute_reply.started":"2023-08-24T19:55:05.304876Z","shell.execute_reply":"2023-08-24T19:55:05.309174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfrecords = glob.glob(\"/kaggle/working/preprocessed/*.tfrecord\")\nprint(\"List of {} TFRecord Files\".format(len(tfrecords)))\n\nwith open(\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\") as f:\n    char_to_num = json.load(f)\n\npad_token = \"P\"\nstart_token = \"<\"\nend_token = \">\"\npad_token_idx = 59\nstart_token_idx = 60\nend_token_idx = 61\n\nchar_to_num[pad_token] = pad_token_idx\nchar_to_num[start_token] = start_token_idx\nchar_to_num[end_token] = end_token_idx\nnum_to_char = {j: i for i, j in char_to_num.items()}","metadata":{"execution":{"iopub.status.busy":"2023-08-24T19:55:05.975241Z","iopub.execute_input":"2023-08-24T19:55:05.975599Z","iopub.status.idle":"2023-08-24T19:55:05.991099Z","shell.execute_reply.started":"2023-08-24T19:55:05.97557Z","shell.execute_reply":"2023-08-24T19:55:05.990022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_pad(x):\n    if tf.shape(x)[0] < FRAME_LEN:\n        x = tf.pad(tensor=x,\n                   paddings=([[0, FRAME_LEN - tf.shape(x)[0]], [0, 0], [0, 0]]),\n                   constant_values=0,\n                   )\n    else:\n        x = tf.image.resize(images=x,\n                            size=(FRAME_LEN, tf.shape(x)[1]),\n                            )\n    return x\n\n\ndef preprocess(x):\n    rhand = tf.gather(x, RHAND_IDX, axis=1)\n    lhand = tf.gather(x, LHAND_IDX, axis=1)\n    rpose = tf.gather(x, RPOSE_IDX, axis=1)\n    lpose = tf.gather(x, LPOSE_IDX, axis=1)\n\n    rnan_idx = tf.reduce_any(tf.math.is_nan(rhand), axis=1)\n    lnan_idx = tf.reduce_any(tf.math.is_nan(lhand), axis=1)\n\n    rnan = tf.math.count_nonzero(rnan_idx)\n    lnan = tf.math.count_nonzero(lnan_idx)\n\n    if rnan > lnan:\n        hand = lhand\n        pose = lpose\n    else:\n        hand = rhand\n        pose = rpose\n\n    hand_x = hand[:, 0 * (len(LHAND_IDX) // 3):1 * (len(LHAND_IDX) // 3)]\n    hand_y = hand[:, 1 * (len(LHAND_IDX) // 3):2 * (len(LHAND_IDX) // 3)]\n    hand_z = hand[:, 2 * (len(LHAND_IDX) // 3):3 * (len(LHAND_IDX) // 3)]\n\n    hand = tf.concat([hand_x[..., tf.newaxis], hand_y[..., tf.newaxis], hand_z[..., tf.newaxis]], axis=-1)\n\n    mean = tf.math.reduce_mean(hand, axis=1)[:, tf.newaxis, :]\n    std = tf.math.reduce_std(hand, axis=1)[:, tf.newaxis, :]\n    hand = (hand - mean) / std\n\n    pose_x = pose[:, 0 * (len(LPOSE_IDX) // 3):1 * (len(LPOSE_IDX) // 3)]\n    pose_y = pose[:, 1 * (len(LPOSE_IDX) // 3):2 * (len(LPOSE_IDX) // 3)]\n    pose_z = pose[:, 2 * (len(LPOSE_IDX) // 3):3 * (len(LPOSE_IDX) // 3)]\n\n    pose = tf.concat([pose_x[..., tf.newaxis], pose_y[..., tf.newaxis], pose_z[..., tf.newaxis]], axis=-1)\n\n    x = tf.concat([hand, pose], axis=1)\n    x = resize_pad(x)\n\n    x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n    x = tf.reshape(x, (FRAME_LEN, len(LHAND_IDX) + len(LPOSE_IDX)))\n\n    return x\n\n\ndef decode_fn(record_bytes):\n    schema = {COL: tf.io.VarLenFeature(dtype=tf.float32) for COL in FEATURE_COLUMNS}\n    schema[\"phrase\"] = tf.io.FixedLenFeature(shape=[], dtype=tf.string)\n    features = tf.io.parse_single_example(record_bytes, features=schema)\n    phrase = features[\"phrase\"]\n    landmarks = ([tf.sparse.to_dense(features[COL]) for COL in FEATURE_COLUMNS])\n    landmarks = tf.transpose(landmarks)\n    return landmarks, phrase\n\n\ntable = tf.lookup.StaticHashTable(\n    initializer=tf.lookup.KeyValueTensorInitializer(\n        keys=list(char_to_num.keys()),\n        values=list(char_to_num.values())\n    ),\n    default_value=tf.constant(-1),\n    name=\"class weights\"\n)\n\n\ndef convert_fn(landmarks, phrase):\n    phrase = start_token + phrase + end_token\n    phrase = tf.strings.bytes_split(phrase)\n    phrase = table.lookup(phrase)\n    phrase = tf.pad(phrase, [[0, 64 - tf.shape(phrase)[0]]], mode=\"CONSTANT\",\n                    constant_values=pad_token_idx)\n    return preprocess(landmarks), phrase","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:01:12.979747Z","iopub.execute_input":"2023-08-24T20:01:12.980114Z","iopub.status.idle":"2023-08-24T20:01:13.005745Z","shell.execute_reply.started":"2023-08-24T20:01:12.980085Z","shell.execute_reply":"2023-08-24T20:01:13.004723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# strategy = tf.distribute.TPUStrategy(resolver)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:01:13.481997Z","iopub.execute_input":"2023-08-24T20:01:13.482376Z","iopub.status.idle":"2023-08-24T20:01:13.486673Z","shell.execute_reply.started":"2023-08-24T20:01:13.482348Z","shell.execute_reply":"2023-08-24T20:01:13.485739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def positional_encoding(length, depth, dtype):\n    depth = depth/2\n    positions = np.arange(length)[:, np.newaxis]\n    depths = np.arange(depth)[np.newaxis, :]/depth\n\n    angle_rates = 1 / (1e4**depths)\n    angle_rads = positions * angle_rates\n\n    pos_encoding = np.concatenate(\n        [np.sin(angle_rads), np.cos(angle_rads)],\n        axis=-1,\n    )\n    return tf.cast(pos_encoding, dtype=dtype)\n\n\nclass PositionalEmbedding(keras.layers.Layer):\n    def __init__(self,\n                 vocab_size: int,\n                 max_len: int,\n                 embed_dim: int,\n                 dtype = tf.int32):\n        super().__init__()\n        self.d_model = embed_dim\n        self.embedding = keras.layers.Embedding(vocab_size, embed_dim)\n        self.pos_encoding = positional_encoding(length=max_len, depth=embed_dim, dtype=dtype)\n\n    def compute_mask(self, *args, **kwargs):\n        return self.embedding.compute_mask(*args, **kwargs)\n\n    def call(self, x, *args, **kwargs):\n        length = tf.shape(x)[1]\n        x = self.embedding(x)\n        x *= tf.math.sqrt(tf.cast(self.d_model, dtype=tf.float32))\n        x = x + self.pos_encoding[tf.newaxis, :length, :]\n        return x\n\n\n# token_emb = PositionalEmbedding(vocab_size=len(char_to_num), max_len=64, embed_dim=512)\n# token_emb(n)\n# print(token_emb(n).shape)\n# print(tf.shape(n))\n\n\nclass LandmarkEmbedding(keras.layers.Layer):\n    def __init__(self, embed_dim=64, pos_embed = False):\n        super().__init__()\n        self.d_model = embed_dim\n        self.pos_embed = pos_embed\n        self.conv1 = layers.Conv1D(\n            embed_dim, 11, strides=2, padding=\"same\", activation=\"gelu\"\n        )\n        self.conv2 = layers.Conv1D(\n            embed_dim, 11, strides=2, padding=\"same\", activation=\"gelu\"\n        )\n        self.conv3 = layers.Conv1D(\n            embed_dim, 11, strides=2, padding=\"same\", activation=\"gelu\"\n        )\n\n    def call(self, x, *args, **kwargs):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        if self.pos_embed == True:\n            pos_emb = positional_encoding(x.shape[1], self.d_model, dtype=tf.float32)\n        else:\n            pos_emb = tf.zeros((tf.shape(x)[1], self.d_model), tf.float32)\n        return x + pos_emb","metadata":{"execution":{"iopub.status.busy":"2023-08-24T22:44:11.76085Z","iopub.execute_input":"2023-08-24T22:44:11.761248Z","iopub.status.idle":"2023-08-24T22:44:11.778349Z","shell.execute_reply.started":"2023-08-24T22:44:11.761216Z","shell.execute_reply":"2023-08-24T22:44:11.777217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BaseAttention(keras.layers.Layer):\n    def __init__(self, *args, **kwargs):\n        super().__init__()\n        self.mha = layers.MultiHeadAttention(**kwargs)\n        self.norm = layers.LayerNormalization()\n        self.add = layers.Add()\n\n\nclass CrossAttention(BaseAttention):\n    def call(self, x, context, *args, **kwargs):\n        attn_output, attn_scores = self.mha(\n            query=x,\n            key=context,\n            value=context,\n            return_attention_scores=True\n        )\n\n        self.last_attn_scores = attn_scores\n\n        x = self.add([x, attn_output])\n        x = self.norm(x)\n\n        return x\n\n\nclass GlobalSelfAttention(BaseAttention):\n    def call(self, x, *args, **kwargs):\n        attn_output = self.mha(\n            query = x,\n            key = x,\n            value =x\n        )\n        x = self.add([x, attn_output])\n        x = self.norm(x)\n        return x\n\n\nclass CausalSelfAttention(BaseAttention):\n    def call(self, x, *args, **kwargs):\n        attn_output = self.mha(\n            query=x,\n            key=x,\n            value=x,\n            use_causal_mask = True,\n        )\n        x = self.add([x, attn_output])\n        x = self.norm(x)\n        return x\n\n\nclass FeedForward(keras.layers.Layer):\n    def __init__(self, embed_dim, dff, dropout_rate=0.1):\n        super().__init__()\n        self.seq = keras.Sequential([\n            layers.Dense(dff, activation=\"relu\"),\n            layers.Dense(embed_dim),\n            layers.Dropout(dropout_rate),\n        ])\n        self.add = layers.Add()\n        self.norm = layers.LayerNormalization()\n\n    def call(self, x, *args, **kwargs):\n        x = self.add([x, self.seq(x)])\n        x = self.norm(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:01:14.222081Z","iopub.execute_input":"2023-08-24T20:01:14.224967Z","iopub.status.idle":"2023-08-24T20:01:14.242879Z","shell.execute_reply.started":"2023-08-24T20:01:14.224928Z","shell.execute_reply":"2023-08-24T20:01:14.241754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderLayer(keras.layers.Layer):\n    def __init__(self, *,\n                 embed_dim:int,\n                 num_heads: int,\n                 dff:int,\n                 dropout_rate=0.1,):\n        super().__init__()\n        self.self_attention = GlobalSelfAttention(num_heads=4,\n                                                  key_dim=384,\n                                                  dropout= .2)\n        self.ffn = FeedForward(embed_dim = 384,\n                               dff= 256,\n                               dropout_rate=.3)\n\n\n    def call(self, x, *args, **kwargs):\n        x = self.self_attention(x)\n        x = self.ffn(x)\n        return x\n\n\nclass Encoder(keras.layers.Layer):\n    def __init__(self, *,embed_dim, num_layers, num_heads,\n                 dff, dropout_rate=0.1):\n        super().__init__()\n        self.d_model = embed_dim\n        self.num_layers = num_layers\n        self.landmark_emb = LandmarkEmbedding(embed_dim=384,\n                                              pos_embed=True)\n        self.enc_layer = [EncoderLayer(\n            embed_dim=embed_dim,\n            num_heads=num_heads,\n            dff=dff,\n            dropout_rate=dropout_rate\n        ) for _ in range(num_layers)]\n        self.dropout = layers.Dropout(dropout_rate)\n\n    def call(self, x, *args, **kwargs):\n        x = self.landmark_emb(x)\n        x = self.dropout(x)\n        for i in range(self.num_layers):\n            x = self.enc_layer[i](x)\n        return x\n\n\nclass DecoderLayer(keras.layers.Layer):\n    def __init__(self, *,\n                 num_heads, embed_dim, dff, dropout_rate=.1):\n        super().__init__()\n        self.causal_attention = CausalSelfAttention(num_heads=2,\n                                                  key_dim= 256,\n                                                  dropout= .2)\n        self.cross_attention = CrossAttention(num_heads= 2,\n                                              key_dim= 256,\n                                              dropout= .2)\n        self.ffn = FeedForward(embed_dim=256,\n                               dff=158,\n                               dropout_rate=.3)\n\n    def call(self, x, context):\n        x = self.causal_attention(x)\n        x = self.cross_attention(x, context)\n\n        self.last_attn_score = self.cross_attention.last_attn_scores\n\n        x = self.ffn(x)\n        return x\n\n\nclass Decoder(keras.layers.Layer):\n    def __init__(self, *, num_layers, num_heads, max_len, embed_dim, dff,\n                 vocab_size, dropout_rate=.1):\n        super().__init__()\n        self.embed_dim = embed_dim\n        self.num_layers = num_layers\n        self.pos_embedding = PositionalEmbedding(vocab_size=vocab_size,\n                                                 max_len=max_len,\n                                                 embed_dim=256,\n                                                 dtype=tf.float32\n                                                 )\n        self.dropout = layers.Dropout(.0)\n        self.dec_layer = [\n            DecoderLayer(embed_dim=embed_dim,\n                         num_heads=num_heads,\n                         dff=dff,\n                         dropout_rate=dropout_rate)\n            for _ in range(num_layers)\n        ]\n        self.last_attention_score = None\n\n    def call(self, x, context):\n        x = self.pos_embedding(x)\n        x = self.dropout(x)\n        for i in range(self.num_layers):\n            x = self.dec_layer[i](x, context)\n        self.last_attention_score = self.dec_layer[-1].last_attn_score\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:23:53.815642Z","iopub.execute_input":"2023-08-24T20:23:53.816233Z","iopub.status.idle":"2023-08-24T20:23:53.835714Z","shell.execute_reply.started":"2023-08-24T20:23:53.816197Z","shell.execute_reply":"2023-08-24T20:23:53.834736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Revisit(keras.layers.Layer):\n    def __init__(self, vocab_size):\n        super().__init__()\n        self.embed_dim = vocab_size\n        self.attn = CrossAttention(num_heads=4, \n                                  key_dim=vocab_size, \n                                  dropout=0.0)\n        self.emb = PositionalEmbedding(vocab_size=vocab_size, \n                                      embed_dim=vocab_size, \n                                      max_len=64, \n                                      dtype=tf.float32)\n        self.corr = [35,49,52,51,42,35,40,50,44,45]\n        \n    def equal_with_multiple_y(self, x, y_values):\n        def cast(inputs):\n            return tf.cast(inputs, dtype=tf.int32)\n    \n        equal_results = [cast(tf.equal(x, y)) for y in y_values]\n        equal_results = layers.Add()(equal_results)\n        return equal_results\n    \n    def call(self, y_true, y_pred):\n        mask_ = tf.cast(equal_with_multiple_y(y_true, self.corr), dtype=tf.float32)\n        y_true = self.emb(y_true)\n        out = self.attn(y_pred, \n                        y_true, \n                        attention_mask=mask_)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-08-24T23:45:41.587282Z","iopub.execute_input":"2023-08-24T23:45:41.587664Z","iopub.status.idle":"2023-08-24T23:45:41.5977Z","shell.execute_reply.started":"2023-08-24T23:45:41.587628Z","shell.execute_reply":"2023-08-24T23:45:41.59656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Transformer(keras.Model):\n    def __init__(self, *,\n                 num_enc_layers = 2,\n                 num_dec_layers = 1,\n                 num_heads = 4,\n                 input_vocab_size = 60,\n                 target_max_len = 100,\n                 embed_dim = 64,\n                 dff = 2048,\n                 dropout_rate = .1):\n        super().__init__()\n        self.num_classes = input_vocab_size\n        self.target_max_len = target_max_len\n        self.encoder = Encoder(num_layers=num_enc_layers,\n                               embed_dim=embed_dim,\n                               num_heads=4,\n                               dff=dff,\n                               dropout_rate=dropout_rate)\n\n        self.decoder = Decoder(num_layers=num_dec_layers,\n                               embed_dim=embed_dim,\n                               num_heads=2,\n                               dff=dff,\n                               vocab_size=input_vocab_size,\n                               max_len=target_max_len,\n                               dropout_rate=dropout_rate)\n        self.final_fcc_layer = layers.Dense(input_vocab_size)\n        self.final_layer = Revisit(vocab_size=input_vocab_size)\n        self.final = layers.Dense(62, activation=\"relu\")\n        self.loss_metric = keras.metrics.Mean(name=\"loss\")\n        self.acc_metric = keras.metrics.Mean(name=\"edit_dist\")\n\n    def call(self, inputs):\n        context, x = inputs\n        context = self.encoder(context)\n        x = self.decoder(x, context)\n        logits = self.final_fcc_layer(x)\n        logits = self.final_layer(inputs[1], logits)\n        logits = self.final(logits)\n\n        try:\n            del logits._keras_mask\n        except AttributeError:\n            pass\n\n        return logits\n\n    @property\n    def metrics(self):\n        return [self.loss_metric]\n\n    def train_step(self, inputs):\n        source = inputs[0]\n        target = inputs[1]\n\n\n        inout_shape = tf.shape(target)\n        batch_size = inout_shape[0]\n\n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        with tf.GradientTape() as tape:\n            preds = self([source, dec_input])\n\n            one_hot = tf.one_hot(dec_target, depth=62)\n            mask = tf.math.logical_not(tf.math.equal(dec_target, pad_token_idx))\n            loss = self.compiled_loss(one_hot, preds, sample_weight= mask)\n        trainable_variables = self.trainable_variables\n        gradients = tape.gradient(loss, trainable_variables)\n        self.optimizer.apply_gradients(zip(gradients, trainable_variables))\n        edit_dist = np.nan\n        try:\n            edit_dist = tf.edit_distance(tf.sparse.from_dense(target),\n                                         tf.sparse.from_dense(tf.cast(tf.argmax(preds, axis=1), tf.int32)), normalize=True)\n            edit_dist = tf.reduce_mean(edit_dist)\n            logging.info(\"Accuracy metric has been calculated\")\n        except TypeError as t:\n            logging.exception(\"training step\")\n\n        self.acc_metric.update_state(edit_dist)\n        self.loss_metric.update_state(loss)\n        return {\"train_loss\": self.loss_metric.result(), \"train_edit_dist\": self.acc_metric.result()}\n\n    def test_step(self, inputs):\n        source= inputs[0]\n        target = inputs[1]\n\n\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n\n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        preds = self([source, dec_input])\n\n        one_hot = tf.one_hot(dec_target, depth=62)\n        mask = tf.math.logical_not(tf.math.equal(dec_target, pad_token_idx))\n        loss = self.compiled_loss(one_hot, preds, sample_weight= mask)\n        edit_dist = np.nan\n        try:\n            edit_dist = tf.edit_distance(tf.sparse.from_dense(target),\n                                         tf.sparse.from_dense(tf.cast(tf.argmax(preds, axis=1), tf.int32)))\n            logging.debug(f\"shape of metric value: {edit_dist.shape}\\n value of metric: {edit_dist}\")\n            edit_dist = tf.reduce_mean(edit_dist)\n        except TypeError as t:\n            logging.exception(\"testing step\")\n\n        self.acc_metric.update_state(edit_dist)\n        self.loss_metric.update_state(loss)\n        return {\"val_loss\": self.loss_metric.result(), \"val_edit_dist\": self.acc_metric.result()}\n\n    def generate(self, source, target_start_token_idx):\n        bs = tf.shape(source)[0]\n        enc = self.encoder(source, training=False)\n        dec_input = tf.ones((bs, 1), tf.int32) * target_start_token_idx\n        dec_logits = []\n        for i in range(self.target_max_len -1):\n            dec_out = self.decoder(dec_input, enc, training=False)\n#             y_true = preprocess(tf.constant([[[target_start_token_idx]]], dtype=tf.float32))\n#             dec_out = self.final_layer(dec_out, y_true, training=False)\n            dec_out = self.final_fcc_layer(dec_out)\n            logits = tf.argmax(dec_out, axis=-1, output_type=tf.int32)\n            last_logit = logits[:, -1][..., tf.newaxis]\n            dec_logits.append(last_logit)\n            dec_input = tf.concat([dec_input, last_logit], axis=-1)\n        return dec_input","metadata":{"execution":{"iopub.status.busy":"2023-08-24T23:53:39.09946Z","iopub.execute_input":"2023-08-24T23:53:39.099869Z","iopub.status.idle":"2023-08-24T23:53:39.128723Z","shell.execute_reply.started":"2023-08-24T23:53:39.099835Z","shell.execute_reply":"2023-08-24T23:53:39.127659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DisplayOutputs(keras.callbacks.Callback):\n    def __init__(self, inputs, idx_to_token, target_start_token_idx=60, target_end_token_idx=61):\n        self.batch = inputs\n        self.idx_to_token = idx_to_token\n        self.target_start_token_idx = target_start_token_idx\n        self.target_end_token_idx = target_end_token_idx\n\n    def on_epoch_end(self, epoch, logs=None):\n        if epoch % 4 != 0:\n            return\n        source = self.batch[0]\n        target = self.batch[1].numpy()\n        bs = tf.shape(source)[0]\n        preds = self.model.generate(source, self.target_start_token_idx)\n        preds = preds.numpy()\n        # bar = tqdm.tqdm(range(bs))\n        for i in range(bs):\n            target_text = \"\".join([self.idx_to_token[_] for _ in target[i, :]])\n            prediction = \"\"\n            for idx in preds[i, :]:\n                # logging.info(f\"prediction index --- {idx}\")\n                # idx_in_range = True if idx in self.idx_to_token else False\n                # if idx_in_range:\n                #     pass\n                # else:\n                #     logging.info(f\"index out of range: {idx}\")\n                #     break\n                prediction += self.idx_to_token[idx]\n                if idx == self.target_end_token_idx:\n                    break\n            print(f\"\\ntarget text:  {target_text.replace('-', '')}\")\n            print(f\"prediction:   {prediction}\\n\")\n\n\nclass Schedular(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, embed_dim, warmup_steps=1000):\n        self.embed_dim = embed_dim\n        self.embed_dim = tf.cast(self.embed_dim, tf.float32)\n\n        self.warmup_steps = warmup_steps\n\n    def __call__(self, step, *args, **kwargs):\n        step = tf.cast(step, tf.float32)\n        arg1 = tf.math.rsqrt(step)\n        arg2 = step * (self.warmup_steps**-1.5)\n        return tf.math.rsqrt(self.embed_dim) * tf.math.minimum(arg2, arg1)\n\n    def get_config(self):\n        config = {\n            \"d_model\": self.embed_dim,\n            \"warmup_steps\": self.warmup_steps\n        }\n        # base_config = super(Schedular, self).get_config()\n        return config","metadata":{"execution":{"iopub.status.busy":"2023-08-24T23:53:39.233312Z","iopub.execute_input":"2023-08-24T23:53:39.234045Z","iopub.status.idle":"2023-08-24T23:53:39.247312Z","shell.execute_reply.started":"2023-08-24T23:53:39.234016Z","shell.execute_reply":"2023-08-24T23:53:39.246204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64\nbatch_size = int(BATCH_SIZE * ((BATCH_SIZE/strategy.num_replicas_in_sync)/BATCH_SIZE))\ntrain_len = int(.8 * len(tfrecords))\n\nAUTO = tf.data.AUTOTUNE\n\ntrain_ds = tf.data.TFRecordDataset(tfrecords[:train_len], num_parallel_reads=AUTO).map(decode_fn)\\\n    .map(convert_fn)\\\n    .shuffle(buffer_size=500).batch(batch_size)\\\n    .prefetch(buffer_size=tf.data.AUTOTUNE).cache()\nval_ds = tf.data.TFRecordDataset(tfrecords[train_len:], num_parallel_reads=AUTO).map(decode_fn)\\\n    .map(convert_fn)\\\n    .batch(batch_size)\\\n    .prefetch(buffer_size=tf.data.AUTOTUNE).cache()\n\ncallback_params = next(iter(val_ds))","metadata":{"execution":{"iopub.status.busy":"2023-08-24T23:53:39.374609Z","iopub.execute_input":"2023-08-24T23:53:39.374951Z","iopub.status.idle":"2023-08-24T23:53:40.574522Z","shell.execute_reply.started":"2023-08-24T23:53:39.374925Z","shell.execute_reply":"2023-08-24T23:53:40.573451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EMBED_DIM = 256\nidx_to_char = list(char_to_num.keys())\n\ncallback_params = next(iter(val_ds))\ndisplay_cb = DisplayOutputs(\n    callback_params, idx_to_char, target_end_token_idx=char_to_num[\">\"], target_start_token_idx=char_to_num[\"<\"]\n)\n# log_dir = \"logs/fit\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n# tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1, update_freq=\"epoch\")# tf.debugging.set_log_device_placement(True)\n# gpus = tf.config.list_logical_devices(\"GPU\")\n# strategy = tf.distribute.MirroredStrategy(gpus)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T23:53:40.576271Z","iopub.execute_input":"2023-08-24T23:53:40.576621Z","iopub.status.idle":"2023-08-24T23:53:40.58618Z","shell.execute_reply.started":"2023-08-24T23:53:40.576589Z","shell.execute_reply":"2023-08-24T23:53:40.585129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nwith strategy.scope():\n    learning_rate = Schedular(EMBED_DIM, 40000)\n    optimizer = keras.optimizers.Adam(0.001, epsilon=1e-9, beta_1=.9, beta_2=.98)\n\n    loss_fn = keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=.1)\n    transformer = Transformer(num_heads=3,\n                              num_enc_layers=2,\n                              num_dec_layers=1,\n                              input_vocab_size=len(char_to_num),\n                              target_max_len=64,\n                              embed_dim=EMBED_DIM,\n                              dff=400,\n                              dropout_rate=0.2)\n\n#     _train_ds = train_ds.take(10).cache()\n#     _val_ds = val_ds.take(10).cache()\n\n    transformer.compile(optimizer=optimizer, loss=loss_fn)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-24T23:53:40.655126Z","iopub.execute_input":"2023-08-24T23:53:40.655794Z","iopub.status.idle":"2023-08-24T23:53:41.912754Z","shell.execute_reply.started":"2023-08-24T23:53:40.655766Z","shell.execute_reply":"2023-08-24T23:53:41.911664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = transformer.fit(train_ds, \n                          validation_data=val_ds,\n                          callbacks=[display_cb],\n                          epochs=100)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-24T23:53:41.914641Z","iopub.execute_input":"2023-08-24T23:53:41.915118Z","iopub.status.idle":"2023-08-24T23:53:57.771884Z","shell.execute_reply.started":"2023-08-24T23:53:41.915082Z","shell.execute_reply":"2023-08-24T23:53:57.770843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" class TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n        self.target_start_token_idx = start_token_idx\n        self.target_end_token_idx = end_token_idx\n        # Load the feature generation and main models\n        self.model = model\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(FEATURE_COLUMNS)], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs, training=False):\n        # Preprocess Data\n        x = tf.cast(inputs, tf.float32)\n        x = x[None]\n        x = tf.cond(tf.shape(x)[1] == 0, lambda: tf.zeros((1, 1, len(FEATURE_COLUMNS))), lambda: tf.identity(x))\n        x = x[0]\n        x = preprocess(x)\n        x = x[None]\n        x = self.model.generate(x, self.target_start_token_idx)\n        x = x[0]\n        idx = tf.argmax(tf.cast(tf.equal(x, self.target_end_token_idx), tf.int32))\n        idx = tf.where(tf.math.less(idx, 1), tf.constant(2, dtype=tf.int64), idx)\n        x = x[1:idx]\n        x = tf.one_hot(x, 59)\n        return {'outputs': x}\n    \ntflitemodel_base = TFLiteModel(transformer)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:03:06.378667Z","iopub.execute_input":"2023-08-24T20:03:06.379074Z","iopub.status.idle":"2023-08-24T20:03:06.716392Z","shell.execute_reply.started":"2023-08-24T20:03:06.379043Z","shell.execute_reply":"2023-08-24T20:03:06.714717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tflitemodel_base(callback_params[0][0])","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:03:07.920303Z","iopub.execute_input":"2023-08-24T20:03:07.920712Z","iopub.status.idle":"2023-08-24T20:03:07.925553Z","shell.execute_reply.started":"2023-08-24T20:03:07.920662Z","shell.execute_reply":"2023-08-24T20:03:07.924128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, tf.lite.OpsSet.SELECT_TF_OPS]\nkeras_model_converter.optimizations = [tf.lite.Optimize.DEFAULT]\nkeras_model_converter.target_spec.supported_types = [tf.float16]\ntflite_model = keras_model_converter.convert()\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \ninfargs = {\"selected_columns\" : FEATURE_COLUMNS}\n\nwith open('inference_args.json', \"w\") as json_file:\n    json.dump(infargs, json_file)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:03:09.714586Z","iopub.execute_input":"2023-08-24T20:03:09.7156Z","iopub.status.idle":"2023-08-24T20:04:15.779223Z","shell.execute_reply.started":"2023-08-24T20:03:09.715566Z","shell.execute_reply":"2023-08-24T20:04:15.778151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transformer.save_weights(\"transfomer.h5\")\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:04:15.781379Z","iopub.execute_input":"2023-08-24T20:04:15.781846Z","iopub.status.idle":"2023-08-24T20:04:16.408653Z","shell.execute_reply.started":"2023-08-24T20:04:15.781812Z","shell.execute_reply":"2023-08-24T20:04:16.407711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip  './model.tflite' './inference_args.json'","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:04:16.410222Z","iopub.execute_input":"2023-08-24T20:04:16.410625Z","iopub.status.idle":"2023-08-24T20:04:18.119051Z","shell.execute_reply.started":"2023-08-24T20:04:16.41059Z","shell.execute_reply":"2023-08-24T20:04:18.117839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tf.lite.Interpreter(\"model.tflite\")\n# interpreter.allocate_tensors()\nREQUIRED_SIGNATURE = \"serving_default\"\nREQUIRED_OUTPUT = \"outputs\"\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\nif REQUIRED_SIGNATURE not in found_signatures:\n    raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=callback_params[0][0])\nprediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\nprint(prediction_str)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:04:18.123828Z","iopub.execute_input":"2023-08-24T20:04:18.124168Z","iopub.status.idle":"2023-08-24T20:04:18.520579Z","shell.execute_reply.started":"2023-08-24T20:04:18.124135Z","shell.execute_reply":"2023-08-24T20:04:18.519434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# callback_params[0][0].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit -r 7 -n 10\n# for frame in callback_params[0]:\n#     output = prediction_fn(inputs=frame)\noutput = prediction_fn(inputs=callback_params[0][0])","metadata":{"execution":{"iopub.status.busy":"2023-08-24T20:04:31.200558Z","iopub.execute_input":"2023-08-24T20:04:31.201277Z","iopub.status.idle":"2023-08-24T20:04:45.449935Z","shell.execute_reply.started":"2023-08-24T20:04:31.201242Z","shell.execute_reply":"2023-08-24T20:04:45.448894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}