{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is an identical copy of [ROHITH's notebook](https://www.kaggle.com/code/irohith/aslfr-ctc-based-on-prev-comp-1st-place), with the sole difference that it loads the weights of the model from my [CTC on TPU](https://www.kaggle.com/code/shlomoron/aslfr-ctc-on-tpu) notebook instead of training it, with the sole purpose to show what is the score that my CTC on TPU gets on the LB.\nIn the forum [there was a discussion](https://www.kaggle.com/competitions/asl-fingerspelling/discussion/426504) about the fact that TPU seems to be behind GPU, probably due to the calculations being in bfloat16, which is less precise than f16/f32. It is perhaps true, as the original notebook achieved a training loss of 3.1931 at epoch 50, versus only 3.9321 for my CTC on TPU version.\nHowever, on the LB, my version achieved the same score (even one point higher, lol. 0.688 vs. 0.687 in the original)\nSo, the bottom line is that TPU achieves the same results as GPU for this model. There may be a noticeable difference with better models, but since such models were not published yet, I can not know.","metadata":{}},{"cell_type":"code","source":"import gc\nimport json\nimport math\nimport pickle\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport tensorflow.keras.mixed_precision as mixed_precision\n\n!pip install Levenshtein\nimport Levenshtein as lev","metadata":{"execution":{"iopub.status.busy":"2023-08-05T23:24:03.403463Z","iopub.execute_input":"2023-08-05T23:24:03.404112Z","iopub.status.idle":"2023-08-05T23:24:44.298905Z","shell.execute_reply.started":"2023-08-05T23:24:03.404074Z","shell.execute_reply":"2023-08-05T23:24:44.297769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n char_to_num = json.load(f)\n\npad_token = '^'\npad_token_idx = 59\n\nchar_to_num[pad_token] = pad_token_idx\n\nnum_to_char = {j:i for i,j in char_to_num.items()}\ndf = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n\nLIP = [\n 61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n 291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n 78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n 95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE] + [f'x_face_{i}' for i in LIP]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE] + [f'y_face_{i}' for i in LIP]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE] + [f'z_face_{i}' for i in LIP]\n\nSEL_COLS = X + Y + Z\nFRAME_LEN = 128\nMAX_PHRASE_LENGTH = 64\n\nLIP_IDX_X = [i for i, col in enumerate(SEL_COLS) if \"face\" in col and \"x\" in col]\nRHAND_IDX_X = [i for i, col in enumerate(SEL_COLS) if \"right\" in col and \"x\" in col]\nLHAND_IDX_X = [i for i, col in enumerate(SEL_COLS) if \"left\" in col and \"x\" in col]\nRPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in RPOSE and \"x\" in col]\nLPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in LPOSE and \"x\" in col]\n\nLIP_IDX_Y = [i for i, col in enumerate(SEL_COLS) if \"face\" in col and \"y\" in col]\nRHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS) if \"right\" in col and \"y\" in col]\nLHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS) if \"left\" in col and \"y\" in col]\nRPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in RPOSE and \"y\" in col]\nLPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in LPOSE and \"y\" in col]\n\nLIP_IDX_Z = [i for i, col in enumerate(SEL_COLS) if \"face\" in col and \"z\" in col]\nRHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS) if \"right\" in col and \"z\" in col]\nLHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS) if \"left\" in col and \"z\" in col]\nRPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in RPOSE and \"z\" in col]\nLPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in LPOSE and \"z\" in col]\n\nRHM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rh_mean.npy\")\nLHM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lh_mean.npy\")\nRPM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rp_mean.npy\")\nLPM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lp_mean.npy\")\nLIPM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lip_mean.npy\")\n\nRHS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rh_std.npy\")\nLHS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lh_std.npy\")\nRPS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rp_std.npy\")\nLPS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lp_std.npy\")\nLIPS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lip_std.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:46:13.809185Z","iopub.execute_input":"2023-07-24T10:46:13.809989Z","iopub.status.idle":"2023-07-24T10:46:14.044158Z","shell.execute_reply.started":"2023-07-24T10:46:13.809949Z","shell.execute_reply":"2023-07-24T10:46:14.043006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n return pd.read_parquet(pq_path, columns=SEL_COLS)\n\nfile_id = df.file_id.iloc[0]\ninpdir = \"/kaggle/input/asl-fingerspelling/train_landmarks\"\npqfile = f\"{inpdir}/{file_id}.parquet\"\nseq_refs = df.loc[df.file_id == file_id]\nseqs = load_relevant_data_subset(pqfile)\n\nseq_id = seq_refs.sequence_id.iloc[0]\nframes = seqs.iloc[seqs.index == seq_id]\nphrase = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:46:17.605383Z","iopub.execute_input":"2023-07-24T10:46:17.605806Z","iopub.status.idle":"2023-07-24T10:46:20.581162Z","shell.execute_reply.started":"2023-07-24T10:46:17.605775Z","shell.execute_reply":"2023-07-24T10:46:20.579991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function()\ndef resize_pad(x):\n if tf.shape(x)[0] < FRAME_LEN:\n x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]), constant_values=float(\"NaN\"))\n else:\n x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n return x\n\n@tf.function(jit_compile=True)\ndef pre_process0(x):\n lip_x = tf.gather(x, LIP_IDX_X, axis=1)\n lip_y = tf.gather(x, LIP_IDX_Y, axis=1)\n lip_z = tf.gather(x, LIP_IDX_Z, axis=1)\n\n rhand_x = tf.gather(x, RHAND_IDX_X, axis=1)\n rhand_y = tf.gather(x, RHAND_IDX_Y, axis=1)\n rhand_z = tf.gather(x, RHAND_IDX_Z, axis=1)\n \n lhand_x = tf.gather(x, LHAND_IDX_X, axis=1)\n lhand_y = tf.gather(x, LHAND_IDX_Y, axis=1)\n lhand_z = tf.gather(x, LHAND_IDX_Z, axis=1)\n\n rpose_x = tf.gather(x, RPOSE_IDX_X, axis=1)\n rpose_y = tf.gather(x, RPOSE_IDX_Y, axis=1)\n rpose_z = tf.gather(x, RPOSE_IDX_Z, axis=1)\n \n lpose_x = tf.gather(x, LPOSE_IDX_X, axis=1)\n lpose_y = tf.gather(x, LPOSE_IDX_Y, axis=1)\n lpose_z = tf.gather(x, LPOSE_IDX_Z, axis=1)\n \n lip = tf.concat([lip_x[..., tf.newaxis], lip_y[..., tf.newaxis], lip_z[..., tf.newaxis]], axis=-1)\n rhand = tf.concat([rhand_x[..., tf.newaxis], rhand_y[..., tf.newaxis], rhand_z[..., tf.newaxis]], axis=-1)\n lhand = tf.concat([lhand_x[..., tf.newaxis], lhand_y[..., tf.newaxis], lhand_z[..., tf.newaxis]], axis=-1)\n rpose = tf.concat([rpose_x[..., tf.newaxis], rpose_y[..., tf.newaxis], rpose_z[..., tf.newaxis]], axis=-1)\n lpose = tf.concat([lpose_x[..., tf.newaxis], lpose_y[..., tf.newaxis], lpose_z[..., tf.newaxis]], axis=-1)\n \n hand = tf.concat([rhand, lhand], axis=1)\n hand = tf.where(tf.math.is_nan(hand), 0.0, hand)\n mask = tf.math.not_equal(tf.reduce_sum(hand, axis=[1, 2]), 0.0)\n\n lip = lip[mask]\n rhand = rhand[mask]\n lhand = lhand[mask]\n rpose = rpose[mask]\n lpose = lpose[mask]\n\n return lip, rhand, lhand, rpose, lpose\n\n@tf.function()\ndef pre_process1(lip, rhand, lhand, rpose, lpose):\n lip = (resize_pad(lip) - LIPM) / LIPS\n rhand = (resize_pad(rhand) - RHM) / RHS\n lhand = (resize_pad(lhand) - LHM) / LHS\n rpose = (resize_pad(rpose) - RPM) / RPS\n lpose = (resize_pad(lpose) - LPM) / LPS\n\n x = tf.concat([lip, rhand, lhand, rpose, lpose], axis=1)\n s = tf.shape(x)\n x = tf.reshape(x, (s[0], s[1]*s[2]))\n x = tf.where(tf.math.is_nan(x), 0.0, x)\n return x\n\npre0 = pre_process0(frames)\npre1 = pre_process1(*pre0)\nINPUT_SHAPE = list(pre1.shape)\nprint(INPUT_SHAPE)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:46:27.827356Z","iopub.execute_input":"2023-07-24T10:46:27.828118Z","iopub.status.idle":"2023-07-24T10:46:30.820741Z","shell.execute_reply.started":"2023-07-24T10:46:27.828082Z","shell.execute_reply":"2023-07-24T10:46:30.819834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_fn(record_bytes):\n schema = {\n \"lip\": tf.io.VarLenFeature(tf.float32),\n \"rhand\": tf.io.VarLenFeature(tf.float32),\n \"lhand\": tf.io.VarLenFeature(tf.float32),\n \"rpose\": tf.io.VarLenFeature(tf.float32),\n \"lpose\": tf.io.VarLenFeature(tf.float32),\n \"phrase\": tf.io.VarLenFeature(tf.int64)\n }\n x = tf.io.parse_single_example(record_bytes, schema)\n\n lip = tf.reshape(tf.sparse.to_dense(x[\"lip\"]), (-1, 40, 3))\n rhand = tf.reshape(tf.sparse.to_dense(x[\"rhand\"]), (-1, 21, 3))\n lhand = tf.reshape(tf.sparse.to_dense(x[\"lhand\"]), (-1, 21, 3))\n rpose = tf.reshape(tf.sparse.to_dense(x[\"rpose\"]), (-1, 5, 3))\n lpose = tf.reshape(tf.sparse.to_dense(x[\"lpose\"]), (-1, 5, 3))\n phrase = tf.sparse.to_dense(x[\"phrase\"])\n\n return lip, rhand, lhand, rpose, lpose, phrase\n\ndef pre_process_fn(lip, rhand, lhand, rpose, lpose, phrase):\n phrase = tf.pad(phrase, [[0, MAX_PHRASE_LENGTH-tf.shape(phrase)[0]]], constant_values=pad_token_idx)\n phrase = tf.cast(phrase, tf.int32)\n return pre_process1(lip, rhand, lhand, rpose, lpose), phrase\n \ntffiles = [f\"/kaggle/input/aslfr-dataset-tfrecords/tfds/{file_id}.tfrecord\" for file_id in df.file_id.unique()]\nval_len = 1#int(0.05 * len(pqfiles))\ntrain_batch_size = 32\nval_batch_size = 32\n\ntrain_dataset = tf.data.TFRecordDataset(tffiles[val_len:]).prefetch(tf.data.AUTOTUNE).shuffle(5000).map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE).map(pre_process_fn, num_parallel_calls=tf.data.AUTOTUNE).batch(train_batch_size).prefetch(tf.data.AUTOTUNE)\nval_dataset = tf.data.TFRecordDataset(tffiles[:val_len]).prefetch(tf.data.AUTOTUNE).map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE).map(pre_process_fn, num_parallel_calls=tf.data.AUTOTUNE).batch(val_batch_size).prefetch(tf.data.AUTOTUNE)\n\nbatch = next(iter(val_dataset))\nbatch[0].shape, batch[1].shape","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:47:20.811174Z","iopub.execute_input":"2023-07-24T10:47:20.811615Z","iopub.status.idle":"2023-07-24T10:47:21.601767Z","shell.execute_reply.started":"2023-07-24T10:47:20.81158Z","shell.execute_reply":"2023-07-24T10:47:21.600466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ECA(tf.keras.layers.Layer):\n def __init__(self, kernel_size=5, **kwargs):\n super().__init__(**kwargs)\n self.supports_masking = True\n self.kernel_size = kernel_size\n self.conv = tf.keras.layers.Conv1D(1, kernel_size=kernel_size, strides=1, padding=\"same\", use_bias=False)\n\n def call(self, inputs, mask=None):\n nn = tf.keras.layers.GlobalAveragePooling1D()(inputs, mask=mask)\n nn = tf.expand_dims(nn, -1)\n nn = self.conv(nn)\n nn = tf.squeeze(nn, -1)\n nn = tf.nn.sigmoid(nn)\n nn = nn[:,None,:]\n return inputs * nn\n\nclass CausalDWConv1D(tf.keras.layers.Layer):\n def __init__(self, \n kernel_size=17,\n dilation_rate=1,\n use_bias=False,\n depthwise_initializer='glorot_uniform',\n name='', **kwargs):\n super().__init__(name=name,**kwargs)\n self.causal_pad = tf.keras.layers.ZeroPadding1D((dilation_rate*(kernel_size-1),0),name=name + '_pad')\n self.dw_conv = tf.keras.layers.DepthwiseConv1D(\n kernel_size,\n strides=1,\n dilation_rate=dilation_rate,\n padding='valid',\n use_bias=use_bias,\n depthwise_initializer=depthwise_initializer,\n name=name + '_dwconv')\n self.supports_masking = True\n \n def call(self, inputs):\n x = self.causal_pad(inputs)\n x = self.dw_conv(x)\n return x\n\ndef Conv1DBlock(channel_size,\n kernel_size,\n dilation_rate=1,\n drop_rate=0.0,\n expand_ratio=2,\n se_ratio=0.25,\n activation='swish',\n name=None):\n '''\n efficient conv1d block, @hoyso48\n '''\n if name is None:\n name = str(tf.keras.backend.get_uid(\"mbblock\"))\n # Expansion phase\n def apply(inputs):\n channels_in = tf.keras.backend.int_shape(inputs)[-1]\n channels_expand = channels_in * expand_ratio\n\n skip = inputs\n\n x = tf.keras.layers.Dense(\n channels_expand,\n use_bias=True,\n activation=activation,\n name=name + '_expand_conv')(inputs)\n\n # Depthwise Convolution\n x = CausalDWConv1D(kernel_size,\n dilation_rate=dilation_rate,\n use_bias=False,\n name=name + '_dwconv')(x)\n\n x = tf.keras.layers.BatchNormalization(momentum=0.95, name=name + '_bn')(x)\n\n x = ECA()(x)\n\n x = tf.keras.layers.Dense(\n channel_size,\n use_bias=True,\n name=name + '_project_conv')(x)\n\n if drop_rate > 0:\n x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1), name=name + '_drop')(x)\n\n if (channels_in == channel_size):\n x = tf.keras.layers.add([x, skip], name=name + '_add')\n return x\n\n return apply\n\nclass MultiHeadSelfAttention(tf.keras.layers.Layer):\n def __init__(self, dim=256, num_heads=4, dropout=0, **kwargs):\n super().__init__(**kwargs)\n self.dim = dim\n self.scale = self.dim ** -0.5\n self.num_heads = num_heads\n self.qkv = tf.keras.layers.Dense(3 * dim, use_bias=False)\n self.drop1 = tf.keras.layers.Dropout(dropout)\n self.proj = tf.keras.layers.Dense(dim, use_bias=False)\n self.supports_masking = True\n\n def call(self, inputs, mask=None):\n qkv = self.qkv(inputs)\n qkv = tf.keras.layers.Permute((2, 1, 3))(tf.keras.layers.Reshape((-1, self.num_heads, self.dim * 3 // self.num_heads))(qkv))\n q, k, v = tf.split(qkv, [self.dim // self.num_heads] * 3, axis=-1)\n\n attn = tf.matmul(q, k, transpose_b=True) * self.scale\n\n if mask is not None:\n mask = mask[:, None, None, :]\n\n attn = tf.keras.layers.Softmax(axis=-1)(attn, mask=mask)\n attn = self.drop1(attn)\n\n x = attn @ v\n x = tf.keras.layers.Reshape((-1, self.dim))(tf.keras.layers.Permute((2, 1, 3))(x))\n x = self.proj(x)\n return x\n\n\ndef TransformerBlock(dim=256, num_heads=6, expand=4, attn_dropout=0.2, drop_rate=0.2, activation='swish'):\n def apply(inputs):\n x = inputs\n x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n x = MultiHeadSelfAttention(dim=dim,num_heads=num_heads,dropout=attn_dropout)(x)\n x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n x = tf.keras.layers.Add()([inputs, x])\n attn_out = x\n\n x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n x = tf.keras.layers.Dense(dim*expand, use_bias=False, activation=activation)(x)\n x = tf.keras.layers.Dense(dim, use_bias=False)(x)\n x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n x = tf.keras.layers.Add()([attn_out, x])\n return x\n return apply\n\ndef positional_encoding(maxlen, num_hid):\n depth = num_hid/2\n positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n angle_rads = tf.linalg.matmul(positions, angle_rates)\n pos_encoding = tf.concat(\n [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n axis=-1)\n return pos_encoding","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:48:15.224359Z","iopub.execute_input":"2023-07-24T10:48:15.224766Z","iopub.status.idle":"2023-07-24T10:48:15.253646Z","shell.execute_reply.started":"2023-07-24T10:48:15.224735Z","shell.execute_reply":"2023-07-24T10:48:15.252525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def CTCLoss(labels, logits):\n label_length = tf.reduce_sum(tf.cast(labels != pad_token_idx, tf.int32), axis=-1)\n logit_length = tf.ones(tf.shape(logits)[0], dtype=tf.int32) * tf.shape(logits)[1]\n loss = tf.nn.ctc_loss(\n labels=labels,\n logits=logits,\n label_length=label_length,\n logit_length=logit_length,\n blank_index=pad_token_idx,\n logits_time_major=False\n )\n loss = tf.reduce_mean(loss)\n return loss","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:48:20.045954Z","iopub.execute_input":"2023-07-24T10:48:20.046919Z","iopub.status.idle":"2023-07-24T10:48:20.052851Z","shell.execute_reply.started":"2023-07-24T10:48:20.046882Z","shell.execute_reply":"2023-07-24T10:48:20.05174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(dim = 384, dropout_step=0):\n inp = tf.keras.Input(INPUT_SHAPE)\n x = inp\n ksize = 11\n\n x = tf.keras.layers.Masking(mask_value=0.0)(inp)\n x = tf.keras.layers.Dense(dim, use_bias=False,name='stem_conv')(x) + positional_encoding(INPUT_SHAPE[0], dim)\n x = tf.keras.layers.BatchNormalization(momentum=0.95,name='stem_bn')(x)\n\n x = Conv1DBlock(dim,11,drop_rate=0.2)(x)\n x = Conv1DBlock(dim,5,drop_rate=0.2)(x)\n x = Conv1DBlock(dim,3,drop_rate=0.2)(x)\n x = TransformerBlock(dim,expand=2)(x)\n\n x = Conv1DBlock(dim,11,drop_rate=0.2)(x)\n x = Conv1DBlock(dim,5,drop_rate=0.2)(x)\n x = Conv1DBlock(dim,3,drop_rate=0.2)(x)\n x = TransformerBlock(dim,expand=2)(x)\n\n x = Conv1DBlock(dim,11,drop_rate=0.2)(x)\n x = Conv1DBlock(dim,5,drop_rate=0.2)(x)\n x = Conv1DBlock(dim,3,drop_rate=0.2)(x)\n x = TransformerBlock(dim,expand=2)(x)\n\n x = tf.keras.layers.Dense(dim*2,activation='relu',name='top_conv')(x)\n x = tf.keras.layers.Dropout(0.4)(x)\n x = tf.keras.layers.Dense(len(char_to_num))(x)\n\n model = tf.keras.Model(inp, x)\n\n loss = CTCLoss\n\n # Adam Optimizer\n optimizer = tfa.optimizers.RectifiedAdam(sma_threshold=4)\n optimizer = tfa.optimizers.Lookahead(optimizer, sync_period=5)\n\n model.compile(loss=loss, optimizer=optimizer)\n\n return model\n\n\ntf.keras.backend.clear_session()\n'''\npolicy = mixed_precision.Policy('float32')\nmixed_precision.set_global_policy(policy)\n\n'''\n \nmodel = get_model()\nmodel(batch[0])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:48:46.26332Z","iopub.execute_input":"2023-07-24T10:48:46.263718Z","iopub.status.idle":"2023-07-24T10:48:50.056625Z","shell.execute_reply.started":"2023-07-24T10:48:46.263687Z","shell.execute_reply":"2023-07-24T10:48:50.055378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_set = [x for x in val_dataset]\n\n@tf.function()\ndef decode_phrase(pred):\n x = tf.argmax(pred, axis=1)\n diff = tf.not_equal(x[:-1], x[1:])\n adjacent_indices = tf.where(diff)[:, 0]\n x = tf.gather(x, adjacent_indices)\n mask = x != pad_token_idx\n x = tf.boolean_mask(x, mask, axis=0)\n return x\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\ndef calculate_val_lev():\n preds = []\n targets = []\n scores = []\n for batch_idx in range(len(val_set)):\n preds_batch = model.predict(val_set[batch_idx][0], verbose = 0)\n targets_batch = val_set[batch_idx][1]\n for pred_idx in range(len(preds_batch)):\n preds.append(\"\".join([rev_character_map.get(s, \"\") for s in decode_phrase(preds_batch[pred_idx]).numpy()]))\n targets.append(\"\".join([rev_character_map.get(s, \"\") for s in targets_batch[pred_idx].numpy()]))\n\n N = [len(phrase) for phrase in targets]\n lev_dist = [lev.distance(preds[i], targets[i]) for i in range(len(targets))]\n print('Lev distance: '+str((np.sum(N) - np.sum(lev_dist))/np.sum(N)))\n\nmodel.load_weights('/kaggle/input/aslfr-ctc-on-tpu/model_epoch_49.h5')\ncalculate_val_lev()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def num_to_char_fn(y):\n return [num_to_char.get(x, \"\") for x in y]\n\n@tf.function()\ndef decode_phrase(pred):\n x = tf.argmax(pred, axis=1)\n diff = tf.not_equal(x[:-1], x[1:])\n adjacent_indices = tf.where(diff)[:, 0]\n x = tf.gather(x, adjacent_indices)\n mask = x != pad_token_idx\n x = tf.boolean_mask(x, mask, axis=0)\n return x\n\n# A utility function to decode the output of the network\ndef decode_batch_predictions(pred):\n output_text = []\n for result in pred:\n result = \"\".join(num_to_char_fn(decode_phrase(result).numpy()))\n output_text.append(result)\n return output_text","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:48:56.687649Z","iopub.execute_input":"2023-07-24T10:48:56.68808Z","iopub.status.idle":"2023-07-24T10:48:56.695788Z","shell.execute_reply.started":"2023-07-24T10:48:56.688046Z","shell.execute_reply":"2023-07-24T10:48:56.694643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# A callback class to output a few transcriptions during training\nclass CallbackEval(tf.keras.callbacks.Callback):\n \"\"\"Displays a batch of outputs after every epoch.\"\"\"\n\n def __init__(self, dataset):\n super().__init__()\n self.dataset = dataset\n\n def on_epoch_end(self, epoch: int, logs=None):\n model.save_weights(\"model.h5\")\n predictions = []\n targets = []\n for batch in self.dataset:\n X, y = batch\n batch_predictions = model(X)\n batch_predictions = decode_batch_predictions(batch_predictions)\n predictions.extend(batch_predictions)\n for label in y:\n label = \"\".join(num_to_char_fn(label.numpy()))\n targets.append(label)\n print(\"-\" * 100)\n # for i in np.random.randint(0, len(predictions), 2):\n for i in range(32):\n print(f\"Target : {targets[i]}\")\n print(f\"Prediction: {predictions[i]}, len: {len(predictions[i])}\")\n print(\"-\" * 100)\n\n# Callback function to check transcription on the val set.\nvalidation_callback = CallbackEval(val_dataset.take(1))","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:49:01.320043Z","iopub.execute_input":"2023-07-24T10:49:01.320452Z","iopub.status.idle":"2023-07-24T10:49:01.333856Z","shell.execute_reply.started":"2023-07-24T10:49:01.320419Z","shell.execute_reply":"2023-07-24T10:49:01.332739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_EPOCHS = 50\nN_WARMUP_EPOCHS = 10\nLR_MAX = 1e-3\nWD_RATIO = 0.05\nWARMUP_METHOD = \"exp\"","metadata":{"execution":{"iopub.status.busy":"2023-07-09T15:10:44.417409Z","iopub.execute_input":"2023-07-09T15:10:44.417753Z","iopub.status.idle":"2023-07-09T15:10:44.426733Z","shell.execute_reply.started":"2023-07-09T15:10:44.417723Z","shell.execute_reply":"2023-07-09T15:10:44.425857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n \n if current_step < num_warmup_steps:\n if WARMUP_METHOD == 'log':\n return lr_max * 0.10 ** (num_warmup_steps - current_step)\n else:\n return lr_max * 2 ** -(num_warmup_steps - current_step)\n else:\n progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max\n \ndef plot_lr_schedule(lr_schedule, epochs):\n fig = plt.figure(figsize=(20, 10))\n plt.plot([None] + lr_schedule + [None])\n # X Labels\n x = np.arange(1, epochs + 1)\n x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n plt.xlim([1, epochs])\n plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n \n # Increase y-limit for better readability\n plt.ylim([0, max(lr_schedule) * 1.1])\n \n # Title\n schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n \n # Plot Learning Rates\n for x, val in enumerate(lr_schedule):\n if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n if x < len(lr_schedule) - 1:\n if lr_schedule[x - 1] < val:\n ha = 'right'\n else:\n ha = 'left'\n elif x == 0:\n ha = 'right'\n else:\n ha = 'left'\n plt.plot(x + 1, val, 'o', color='black');\n offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n \n plt.xlabel('Epoch', size=16, labelpad=5)\n plt.ylabel('Learning Rate', size=16, labelpad=5)\n plt.grid()\n plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=0)\n\n# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n def __init__(self, wd_ratio=WD_RATIO):\n self.step_counter = 0\n self.wd_ratio = wd_ratio\n \n def on_epoch_begin(self, epoch, logs=None):\n model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"execution":{"iopub.status.busy":"2023-07-09T15:10:44.428048Z","iopub.execute_input":"2023-07-09T15:10:44.428352Z","iopub.status.idle":"2023-07-09T15:10:45.112403Z","shell.execute_reply.started":"2023-07-09T15:10:44.428322Z","shell.execute_reply":"2023-07-09T15:10:45.111353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nhistory = model.fit(\n train_dataset,\n validation_data=val_dataset,\n epochs=N_EPOCHS,\n callbacks=[\n validation_callback,\n lr_callback,\n WeightDecayCallback(),\n ]\n)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-07-09T15:10:59.488603Z","iopub.execute_input":"2023-07-09T15:10:59.488998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TFLiteModel(tf.Module):\n def __init__(self, model):\n super(TFLiteModel, self).__init__()\n self.model = model\n \n @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(SEL_COLS)], dtype=tf.float32, name='inputs')])\n def __call__(self, inputs, training=False):\n # Preprocess Data\n x = tf.cast(inputs, tf.float32)\n x = x[None]\n x = tf.cond(tf.shape(x)[1] == 0, lambda: tf.zeros((1, 1, len(SEL_COLS))), lambda: tf.identity(x))\n x = x[0]\n x = pre_process0(x)\n x = pre_process1(*x)\n x = tf.reshape(x, INPUT_SHAPE)\n x = x[None]\n x = self.model(x, training=False)\n x = x[0]\n x = decode_phrase(x)\n x = tf.cond(tf.shape(x)[0] == 0, lambda: tf.zeros(1, tf.int32), lambda: tf.cast(tf.identity(x), tf.int32))\n x = tf.one_hot(x, 59)\n return {'outputs': x}\n\ntflitemodel_base = TFLiteModel(model)\ntflitemodel_base(frames)[\"outputs\"].shape","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:49:29.892472Z","iopub.execute_input":"2023-07-24T10:49:29.892885Z","iopub.status.idle":"2023-07-24T10:49:33.090954Z","shell.execute_reply.started":"2023-07-24T10:49:29.892856Z","shell.execute_reply":"2023-07-24T10:49:33.089881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]#, tf.lite.OpsSet.SELECT_TF_OPS]\ntflite_model = keras_model_converter.convert()\nwith open('model.tflite', 'wb') as f:\n f.write(tflite_model)\n \nwith open('inference_args.json', \"w\") as f:\n json.dump({\"selected_columns\" : SEL_COLS}, f)\n \n!zip submission.zip './model.tflite' './inference_args.json'","metadata":{"execution":{"iopub.status.busy":"2023-07-24T10:49:38.684047Z","iopub.execute_input":"2023-07-24T10:49:38.684451Z","iopub.status.idle":"2023-07-24T10:50:25.520324Z","shell.execute_reply.started":"2023-07-24T10:49:38.684406Z","shell.execute_reply":"2023-07-24T10:50:25.5189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"inference_args.json\", \"r\") as f:\n SEL_COLS = json.load(f)[\"selected_columns\"]\n \ndef load_relevant_data_subset(pq_path):\n return pd.read_parquet(pq_path, columns=SEL_COLS)\n\ndef create_data_gen(file_ids, y_mul=1):\n def gen():\n for file_id in file_ids:\n pqfile = f\"{inpdir}/{file_id}.parquet\"\n seq_refs = df.loc[df.file_id == file_id]\n seqs = load_relevant_data_subset(pqfile)\n\n for seq_id in seq_refs.sequence_id:\n x = seqs.iloc[seqs.index == seq_id].to_numpy()\n y = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])\n \n r_nonan = np.sum(np.sum(np.isnan(x[:, RHAND_IDX_X]), axis = 1) == 0)\n l_nonan = np.sum(np.sum(np.isnan(x[:, LHAND_IDX_X]), axis = 1) == 0)\n no_nan = max(r_nonan, l_nonan)\n \n if y_mul*len(y)