{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install mediapipe\n!pip install tensorflow","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-02T03:27:59.089753Z","iopub.execute_input":"2023-08-02T03:27:59.090161Z","iopub.status.idle":"2023-08-02T03:28:26.436567Z","shell.execute_reply.started":"2023-08-02T03:27:59.09012Z","shell.execute_reply":"2023-08-02T03:28:26.434998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nimport tensorflow as tf\nimport json\nimport mediapipe\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport random\nimport tensorflow\ntensorflow.random.set_seed(27)\nimport seaborn as sns\nsns.set()\n\nfrom itertools import product\nfrom math import ceil\nfrom string import ascii_uppercase\nfrom time import time\nfrom keras import backend as K\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras.layers import Dense, Conv2D, MaxPool2D, Flatten, Dropout, BatchNormalization\nfrom keras.models import Sequential, load_model\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom skimage.transform import resize\nfrom mediapipe.framework.formats import landmark_pb2\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tqdm.notebook import tqdm\nfrom matplotlib import animation, rc\n","metadata":{"execution":{"iopub.status.busy":"2023-08-02T03:31:21.539655Z","iopub.execute_input":"2023-08-02T03:31:21.54024Z","iopub.status.idle":"2023-08-02T03:31:33.255128Z","shell.execute_reply.started":"2023-08-02T03:31:21.540197Z","shell.execute_reply":"2023-08-02T03:31:33.253821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"TensorFlow v\" + tf.__version__)\nprint(\"Mediapipe v\" + mediapipe.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:44.4817Z","iopub.execute_input":"2023-07-23T15:59:44.482031Z","iopub.status.idle":"2023-07-23T15:59:44.492177Z","shell.execute_reply.started":"2023-07-23T15:59:44.481996Z","shell.execute_reply":"2023-07-23T15:59:44.491291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_csv_data(path):\n#     dataframe = pd.read_csv(path)\n\n#     labels = dataframe['label'].values\n#     labels_categorical = to_categorical(labels)\n#     dataframe.drop('label', axis=1, inplace = True)\n\n#     images = dataframe.values\n#     images = images / 255\n#     images = np.array([np.reshape(i, (28, 28, 1)) for i in images])\n\n#     return images, labels, labels_categorical","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_images, train_labels, train_labels_categorical = get_csv_data('/kaggle/input/asl-fingerspelling/train.csv')\n# test_images, test_labels, test_labels_categorical = get_csv_data('/content/sign_mnist_test.csv')\n\ndataset_df = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\nprint(\"Full train dataset shape is {}\".format(dataset_df.shape))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-23T15:59:44.495487Z","iopub.execute_input":"2023-07-23T15:59:44.495836Z","iopub.status.idle":"2023-07-23T15:59:44.633457Z","shell.execute_reply.started":"2023-07-23T15:59:44.495804Z","shell.execute_reply":"2023-07-23T15:59:44.632489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:44.635263Z","iopub.execute_input":"2023-07-23T15:59:44.635968Z","iopub.status.idle":"2023-07-23T15:59:44.649961Z","shell.execute_reply.started":"2023-07-23T15:59:44.635933Z","shell.execute_reply":"2023-07-23T15:59:44.648952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_id, file_id, phrase = dataset_df.iloc[0][['sequence_id', 'file_id', 'phrase']]\nprint(f\"sequence_id: {sequence_id}, file_id: {file_id}, phrase: {phrase}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:44.651287Z","iopub.execute_input":"2023-07-23T15:59:44.651813Z","iopub.status.idle":"2023-07-23T15:59:44.658773Z","shell.execute_reply.started":"2023-07-23T15:59:44.651781Z","shell.execute_reply":"2023-07-23T15:59:44.65773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sequence_df = pq.read_table(f\"/kaggle/input/asl-fingerspelling/train_landmarks/{str(file_id)}.parquet\",\n    filters=[[('sequence_id', '=', sequence_id)],]).to_pandas()\nprint(\"Full sequence dataset shape is {}\".format(sample_sequence_df.shape))","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:44.660388Z","iopub.execute_input":"2023-07-23T15:59:44.661034Z","iopub.status.idle":"2023-07-23T15:59:49.031248Z","shell.execute_reply.started":"2023-07-23T15:59:44.661001Z","shell.execute_reply":"2023-07-23T15:59:49.029942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sequence_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:49.033336Z","iopub.execute_input":"2023-07-23T15:59:49.033951Z","iopub.status.idle":"2023-07-23T15:59:49.05962Z","shell.execute_reply.started":"2023-07-23T15:59:49.033916Z","shell.execute_reply":"2023-07-23T15:59:49.058048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set global configuration settings\nmatplotlib.rcParams['animation.embed_limit'] = 2**128\nmatplotlib.rcParams['savefig.pad_inches'] = 0\n\n# Create figure and axes objects\nfig = plt.figure(figsize=(6, 9))\nax = fig.add_axes([0., 0., 1., 1.])\nax.set_axis_off()\n\ndef create_animation(images):\n    im = ax.imshow(images[0], cmap=\"gray\")\n    plt.close(fig)\n\n    def animate_func(i):\n        im.set_array(images[i])\n        return [im]\n\n    return animation.FuncAnimation(fig, animate_func, frames=len(images), interval=1000/10)\n\n\n\n#Function create animation from images.\n# matplotlib.rcParams['animation.embed_limit'] = 2**128\n# matplotlib.rcParams['savefig.pad_inches'] = 0\n# rc('animation', html='jshtml')\n\n# def create_animation(images):\n#     fig = plt.figure(figsize=(6, 9))\n#     ax = plt.Axes(fig, [0., 0., 1., 1.])\n#     ax.set_axis_off()\n#     fig.add_axes(ax)\n#     im=ax.imshow(images[0], cmap=\"gray\")\n#     plt.close(fig)\n    \n#     def animate_func(i):\n#         im.set_array(images[i])\n#         return [im]\n\n#     return animation.FuncAnimation(fig, animate_func, frames=len(images), interval=1000/10)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:49.061549Z","iopub.execute_input":"2023-07-23T15:59:49.062226Z","iopub.status.idle":"2023-07-23T15:59:49.159021Z","shell.execute_reply.started":"2023-07-23T15:59:49.062191Z","shell.execute_reply":"2023-07-23T15:59:49.157989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract the landmark data and convert it to an image using medipipe library.\n# This function extracts the data for both hands.\n\nmp_pose = mediapipe.solutions.pose\nmp_hands = mediapipe.solutions.hands\nmp_drawing = mediapipe.solutions.drawing_utils \nmp_drawing_styles = mediapipe.solutions.drawing_styles\n\ndef get_hands(seq_df):\n    images = []\n    all_hand_landmarks = []\n    for seq_idx in range(len(seq_df)):\n        x_hand = seq_df.iloc[seq_idx].filter(regex=\"x_right_hand.*\").values\n        y_hand = seq_df.iloc[seq_idx].filter(regex=\"y_right_hand.*\").values\n        z_hand = seq_df.iloc[seq_idx].filter(regex=\"z_right_hand.*\").values\n\n        right_hand_image = np.zeros((600, 600, 3))\n\n        right_hand_landmarks = landmark_pb2.NormalizedLandmarkList()\n        \n        for x, y, z in zip(x_hand, y_hand, z_hand):\n            right_hand_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n                right_hand_image,\n                right_hand_landmarks,\n                mp_hands.HAND_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        x_hand = seq_df.iloc[seq_idx].filter(regex=\"x_left_hand.*\").values\n        y_hand = seq_df.iloc[seq_idx].filter(regex=\"y_left_hand.*\").values\n        z_hand = seq_df.iloc[seq_idx].filter(regex=\"z_left_hand.*\").values\n        \n        left_hand_image = np.zeros((600, 600, 3))\n        \n        left_hand_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for x, y, z in zip(x_hand, y_hand, z_hand):\n            left_hand_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n                left_hand_image,\n                left_hand_landmarks,\n                mp_hands.HAND_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        images.append([right_hand_image.astype(np.uint8), left_hand_image.astype(np.uint8)])\n        all_hand_landmarks.append([right_hand_landmarks, left_hand_landmarks])\n    return images, all_hand_landmarks","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:49.16436Z","iopub.execute_input":"2023-07-23T15:59:49.165013Z","iopub.status.idle":"2023-07-23T15:59:49.184398Z","shell.execute_reply.started":"2023-07-23T15:59:49.164981Z","shell.execute_reply":"2023-07-23T15:59:49.183219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the images created using mediapipe apis\nhand_images, hand_landmarks = get_hands(sample_sequence_df)\n# Fetch and show the data for right hand\ncreate_animation(np.array(hand_images)[:, 0])","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:49.185985Z","iopub.execute_input":"2023-07-23T15:59:49.186567Z","iopub.status.idle":"2023-07-23T15:59:52.384499Z","shell.execute_reply.started":"2023-07-23T15:59:49.186536Z","shell.execute_reply":"2023-07-23T15:59:52.383289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pose coordinates for hand movement.\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:52.386337Z","iopub.execute_input":"2023-07-23T15:59:52.386767Z","iopub.status.idle":"2023-07-23T15:59:52.395675Z","shell.execute_reply.started":"2023-07-23T15:59:52.38673Z","shell.execute_reply":"2023-07-23T15:59:52.393849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create x,y,z label names from coordinates\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE]\n\nFEATURE_COLUMNS = X + Y + Z","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:52.39885Z","iopub.execute_input":"2023-07-23T15:59:52.399696Z","iopub.status.idle":"2023-07-23T15:59:52.420074Z","shell.execute_reply.started":"2023-07-23T15:59:52.399651Z","shell.execute_reply":"2023-07-23T15:59:52.417111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"x_\" in col]\nY_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"y_\" in col]\nZ_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"z_\" in col]\n\nRHAND_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"right\" in col]\nLHAND_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if  \"left\" in col]\nRPOSE_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if  \"pose\" in col and int(col[-2:]) in RPOSE]\nLPOSE_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if  \"pose\" in col and int(col[-2:]) in LPOSE]","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:52.422646Z","iopub.execute_input":"2023-07-23T15:59:52.423149Z","iopub.status.idle":"2023-07-23T15:59:52.440674Z","shell.execute_reply.started":"2023-07-23T15:59:52.423104Z","shell.execute_reply":"2023-07-23T15:59:52.438844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set length of frames to 128\nFRAME_LEN = 128\n\n# Create directory to store the new data\nif not os.path.isdir(\"preprocessed\"):\n    os.mkdir(\"preprocessed\")\nelse:\n    shutil.rmtree(\"preprocessed\")\n    os.mkdir(\"preprocessed\")\n\n# Loop through each file_id\nfor file_id in tqdm(dataset_df.file_id.unique()):\n    # Parquet file name\n    pq_file = f\"/kaggle/input/asl-fingerspelling/train_landmarks/{file_id}.parquet\"\n    # Filter train.csv and fetch entries only for the relevant file_id\n    file_df = dataset_df.loc[dataset_df[\"file_id\"] == file_id]\n    # Fetch the parquet file\n    parquet_df = pq.read_table(f\"/kaggle/input/asl-fingerspelling/train_landmarks/{str(file_id)}.parquet\",\n                              columns=['sequence_id'] + FEATURE_COLUMNS).to_pandas()\n    # File name for the updated data\n    tf_file = f\"preprocessed/{file_id}.tfrecord\"\n    parquet_numpy = parquet_df.to_numpy()\n    # Initialize the pointer to write the output of \n    # each `for loop` below as a sequence into the file.\n    with tf.io.TFRecordWriter(tf_file) as file_writer:\n        # Loop through each sequence in file.\n        for seq_id, phrase in zip(file_df.sequence_id, file_df.phrase):\n            # Fetch sequence data\n            frames = parquet_numpy[parquet_df.index == seq_id]\n            \n            # Calculate the number of NaN values in each hand landmark\n            r_nonan = np.sum(np.sum(np.isnan(frames[:, RHAND_IDX]), axis = 1) == 0)\n            l_nonan = np.sum(np.sum(np.isnan(frames[:, LHAND_IDX]), axis = 1) == 0)\n            no_nan = max(r_nonan, l_nonan)\n            \n            if 2*len(phrase)<no_nan:\n                features = {FEATURE_COLUMNS[i]: tf.train.Feature(\n                    float_list=tf.train.FloatList(value=frames[:, i])) for i in range(len(FEATURE_COLUMNS))}\n                features[\"phrase\"] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[bytes(phrase, 'utf-8')]))\n                record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n                file_writer.write(record_bytes)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T15:59:52.442893Z","iopub.execute_input":"2023-07-23T15:59:52.443665Z","iopub.status.idle":"2023-07-23T16:09:49.052223Z","shell.execute_reply.started":"2023-07-23T15:59:52.443627Z","shell.execute_reply":"2023-07-23T16:09:49.051299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get the saved TFRecord files into a list\ntf_records = dataset_df.file_id.map(lambda x: f'/kaggle/working/preprocessed/{x}.tfrecord').unique()\nprint(f\"List of {len(tf_records)} TFRecord files.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:49.054005Z","iopub.execute_input":"2023-07-23T16:09:49.054735Z","iopub.status.idle":"2023-07-23T16:09:49.109566Z","shell.execute_reply.started":"2023-07-23T16:09:49.054696Z","shell.execute_reply":"2023-07-23T16:09:49.108305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\n# Add pad_token, start pointer and end pointer to the dict\npad_token = 'P'\nstart_token = '<'\nend_token = '>'\npad_token_idx = 59\nstart_token_idx = 60\nend_token_idx = 61\n\nchar_to_num[pad_token] = pad_token_idx\nchar_to_num[start_token] = start_token_idx\nchar_to_num[end_token] = end_token_idx\nnum_to_char = {j:i for i,j in char_to_num.items()}","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:49.110841Z","iopub.execute_input":"2023-07-23T16:09:49.11119Z","iopub.status.idle":"2023-07-23T16:09:49.122403Z","shell.execute_reply.started":"2023-07-23T16:09:49.111148Z","shell.execute_reply":"2023-07-23T16:09:49.121523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reference: https://www.kaggle.com/code/irohith/aslfr-transformer/notebook\n\n# Function to resize and add padding.\ndef resize_pad(x):\n    if tf.shape(x)[0] < FRAME_LEN:\n        x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]))\n    else:\n        x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n    return x\n\n# Detect the dominant hand from the number of NaN values.\n# Dominant hand will have less NaN values since it is in frame moving.\ndef pre_process(x):\n    rhand = tf.gather(x, RHAND_IDX, axis=1)\n    lhand = tf.gather(x, LHAND_IDX, axis=1)\n    rpose = tf.gather(x, RPOSE_IDX, axis=1)\n    lpose = tf.gather(x, LPOSE_IDX, axis=1)\n    \n    rnan_idx = tf.reduce_any(tf.math.is_nan(rhand), axis=1)\n    lnan_idx = tf.reduce_any(tf.math.is_nan(lhand), axis=1)\n    \n    rnans = tf.math.count_nonzero(rnan_idx)\n    lnans = tf.math.count_nonzero(lnan_idx)\n    \n    # For dominant hand\n    if rnans > lnans:\n        hand = lhand\n        pose = lpose\n        \n        hand_x = hand[:, 0*(len(LHAND_IDX)//3) : 1*(len(LHAND_IDX)//3)]\n        hand_y = hand[:, 1*(len(LHAND_IDX)//3) : 2*(len(LHAND_IDX)//3)]\n        hand_z = hand[:, 2*(len(LHAND_IDX)//3) : 3*(len(LHAND_IDX)//3)]\n        hand = tf.concat([1-hand_x, hand_y, hand_z], axis=1)\n        \n        pose_x = pose[:, 0*(len(LPOSE_IDX)//3) : 1*(len(LPOSE_IDX)//3)]\n        pose_y = pose[:, 1*(len(LPOSE_IDX)//3) : 2*(len(LPOSE_IDX)//3)]\n        pose_z = pose[:, 2*(len(LPOSE_IDX)//3) : 3*(len(LPOSE_IDX)//3)]\n        pose = tf.concat([1-pose_x, pose_y, pose_z], axis=1)\n    else:\n        hand = rhand\n        pose = rpose\n    \n    hand_x = hand[:, 0*(len(LHAND_IDX)//3) : 1*(len(LHAND_IDX)//3)]\n    hand_y = hand[:, 1*(len(LHAND_IDX)//3) : 2*(len(LHAND_IDX)//3)]\n    hand_z = hand[:, 2*(len(LHAND_IDX)//3) : 3*(len(LHAND_IDX)//3)]\n    hand = tf.concat([hand_x[..., tf.newaxis], hand_y[..., tf.newaxis], hand_z[..., tf.newaxis]], axis=-1)\n    \n    mean = tf.math.reduce_mean(hand, axis=1)[:, tf.newaxis, :]\n    std = tf.math.reduce_std(hand, axis=1)[:, tf.newaxis, :]\n    hand = (hand - mean) / std\n\n    pose_x = pose[:, 0*(len(LPOSE_IDX)//3) : 1*(len(LPOSE_IDX)//3)]\n    pose_y = pose[:, 1*(len(LPOSE_IDX)//3) : 2*(len(LPOSE_IDX)//3)]\n    pose_z = pose[:, 2*(len(LPOSE_IDX)//3) : 3*(len(LPOSE_IDX)//3)]\n    pose = tf.concat([pose_x[..., tf.newaxis], pose_y[..., tf.newaxis], pose_z[..., tf.newaxis]], axis=-1)\n    \n    x = tf.concat([hand, pose], axis=1)\n    x = resize_pad(x)\n    \n    x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n    x = tf.reshape(x, (FRAME_LEN, len(LHAND_IDX) + len(LPOSE_IDX)))\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:49.124059Z","iopub.execute_input":"2023-07-23T16:09:49.124934Z","iopub.status.idle":"2023-07-23T16:09:49.144732Z","shell.execute_reply.started":"2023-07-23T16:09:49.124903Z","shell.execute_reply":"2023-07-23T16:09:49.143783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the schema as a constant outside the function\nschema = {COL: tf.io.VarLenFeature(dtype=tf.float32) for COL in FEATURE_COLUMNS}\nschema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n\ndef decode_fn(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, schema)\n    phrase = features[\"phrase\"]\n    \n    # Stack the features directly instead of using list comprehension\n    landmarks = tf.stack([tf.sparse.to_dense(features[COL]) for COL in FEATURE_COLUMNS], axis=1)\n\n    return landmarks, phrase\n\n# def decode_fn(record_bytes):\n#     schema = {COL: tf.io.VarLenFeature(dtype=tf.float32) for COL in FEATURE_COLUMNS}\n#     schema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n#     features = tf.io.parse_single_example(record_bytes, schema)\n#     phrase = features[\"phrase\"]\n#     landmarks = ([tf.sparse.to_dense(features[COL]) for COL in FEATURE_COLUMNS])\n#     # Transpose to maintain the original shape of landmarks data.\n#     landmarks = tf.transpose(landmarks)\n    \n#     return landmarks, phrase","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:49.145962Z","iopub.execute_input":"2023-07-23T16:09:49.14671Z","iopub.status.idle":"2023-07-23T16:09:49.158925Z","shell.execute_reply.started":"2023-07-23T16:09:49.146678Z","shell.execute_reply":"2023-07-23T16:09:49.158064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table = tf.lookup.StaticHashTable(\n    initializer=tf.lookup.KeyValueTensorInitializer(\n        keys=list(char_to_num.keys()),\n        values=list(char_to_num.values()),\n    ),\n    default_value=tf.constant(-1),\n    name=\"class_weight\"\n)\n\n# def convert_fn(landmarks, phrase):\n#     # Add start and end pointers to phrase.\n#     phrase = start_token + phrase + end_token\n#     phrase = tf.strings.bytes_split(phrase)\n#     phrase = table.lookup(phrase)\n#     # Vectorize and add padding.\n#     phrase = tf.pad(phrase, paddings=[[0, 64 - tf.shape(phrase)[0]]], mode = 'CONSTANT',\n#                     constant_values = pad_token_idx)\n#     # Apply pre_process function to the landmarks.\n#     return pre_process(landmarks), phrase\n\n\ndef convert_fn(landmarks, phrase):\n    # Add start and end pointers to phrase, and convert to Unicode.\n    phrase = tf.strings.unicode_split(start_token + phrase + end_token, input_encoding='UTF-8')\n\n    # Look up the tokens in the table.\n    phrase = table.lookup(phrase)\n\n    # Calculate the shape of the phrase tensor.\n    phrase_shape = tf.shape(phrase)[0]\n\n    # Pad the phrase tensor to a fixed length.\n    phrase = tf.pad(phrase, paddings=[[0, 64 - phrase_shape]], mode='CONSTANT', constant_values=pad_token_idx)\n\n    # Apply the pre_process function to the landmarks.\n    landmarks = pre_process(landmarks)\n\n    return landmarks, phrase","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:49.160556Z","iopub.execute_input":"2023-07-23T16:09:49.160956Z","iopub.status.idle":"2023-07-23T16:09:49.173379Z","shell.execute_reply.started":"2023-07-23T16:09:49.160926Z","shell.execute_reply":"2023-07-23T16:09:49.172413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\ntrain_len = int(0.8 * len(tf_records))\n\ntrain_ds = tf.data.TFRecordDataset(tf_records[:train_len]).map(decode_fn).map(convert_fn).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE).cache()\nvalid_ds = tf.data.TFRecordDataset(tf_records[train_len:]).map(decode_fn).map(convert_fn).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE).cache()","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:49.175154Z","iopub.execute_input":"2023-07-23T16:09:49.175628Z","iopub.status.idle":"2023-07-23T16:09:50.650691Z","shell.execute_reply.started":"2023-07-23T16:09:49.175597Z","shell.execute_reply":"2023-07-23T16:09:50.649742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TokenEmbedding(layers.Layer):\n    def __init__(self, num_vocab=1000, maxlen=100, num_hid=64):\n        super().__init__()\n        self.emb = tf.keras.layers.Embedding(num_vocab, num_hid)\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=num_hid)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        x = self.emb(x)\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        return x + positions\n\n\nclass LandmarkEmbedding(layers.Layer):\n    def __init__(self, num_hid=64, maxlen=100):\n        super().__init__()\n        self.conv1 = tf.keras.layers.Conv1D(\n            num_hid, 11, strides=2, padding=\"same\", activation=\"relu\"\n        )\n        self.conv2 = tf.keras.layers.Conv1D(\n            num_hid, 11, strides=2, padding=\"same\", activation=\"relu\"\n        )\n        self.conv3 = tf.keras.layers.Conv1D(\n            num_hid, 11, strides=2, padding=\"same\", activation=\"relu\"\n        )\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=num_hid)\n\n    def call(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        return self.conv3(x)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:50.653572Z","iopub.execute_input":"2023-07-23T16:09:50.653997Z","iopub.status.idle":"2023-07-23T16:09:50.664031Z","shell.execute_reply.started":"2023-07-23T16:09:50.653959Z","shell.execute_reply":"2023-07-23T16:09:50.662898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerEncoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, rate=0.1):\n        super().__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dense(embed_dim),\n            ]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:50.665528Z","iopub.execute_input":"2023-07-23T16:09:50.666145Z","iopub.status.idle":"2023-07-23T16:09:50.679398Z","shell.execute_reply.started":"2023-07-23T16:09:50.666112Z","shell.execute_reply":"2023-07-23T16:09:50.678484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Customized to add `training` variable\n# Reference: https://www.kaggle.com/code/shlomoron/aslfr-a-simple-transformer/notebook\n\nclass TransformerDecoder(layers.Layer):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, dropout_rate=0.1):\n        super().__init__()\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = layers.LayerNormalization(epsilon=1e-6)\n        self.self_att = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=embed_dim\n        )\n        self.enc_att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.self_dropout = layers.Dropout(0.5)\n        self.enc_dropout = layers.Dropout(0.1)\n        self.ffn_dropout = layers.Dropout(0.1)\n        self.ffn = keras.Sequential(\n            [\n                layers.Dense(feed_forward_dim, activation=\"relu\"),\n                layers.Dense(embed_dim),\n            ]\n        )\n\n    def causal_attention_mask(self, batch_size, n_dest, n_src, dtype):\n        \"\"\"Masks the upper half of the dot product matrix in self attention.\n\n        This prevents flow of information from future tokens to current token.\n        1's in the lower triangle, counting from the lower right corner.\n        \"\"\"\n        i = tf.range(n_dest)[:, None]\n        j = tf.range(n_src)\n        m = i >= j - n_src + n_dest\n        mask = tf.cast(m, dtype)\n        mask = tf.reshape(mask, [1, n_dest, n_src])\n        mult = tf.concat(\n            [batch_size[..., tf.newaxis], tf.constant([1, 1], dtype=tf.int32)], 0\n        )\n        return tf.tile(mask, mult)\n\n    def call(self, enc_out, target, training):\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        seq_len = input_shape[1]\n        causal_mask = self.causal_attention_mask(batch_size, seq_len, seq_len, tf.bool)\n        target_att = self.self_att(target, target, attention_mask=causal_mask)\n        target_norm = self.layernorm1(target + self.self_dropout(target_att, training = training))\n        enc_out = self.enc_att(target_norm, enc_out)\n        enc_out_norm = self.layernorm2(self.enc_dropout(enc_out, training = training) + target_norm)\n        ffn_out = self.ffn(enc_out_norm)\n        ffn_out_norm = self.layernorm3(enc_out_norm + self.ffn_dropout(ffn_out, training = training))\n        return ffn_out_norm","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:50.682588Z","iopub.execute_input":"2023-07-23T16:09:50.682876Z","iopub.status.idle":"2023-07-23T16:09:50.697138Z","shell.execute_reply.started":"2023-07-23T16:09:50.682854Z","shell.execute_reply":"2023-07-23T16:09:50.696257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Customized to add edit_dist metric and training variable.\n# Reference:\n# https://www.kaggle.com/code/irohith/aslfr-transformer/notebook\n# https://www.kaggle.com/code/shlomoron/aslfr-a-simple-transformer/notebook\n\nclass Transformer(keras.Model):\n    def __init__(\n        self,\n        num_hid=64,\n        num_head=2,\n        num_feed_forward=128,\n        source_maxlen=100,\n        target_maxlen=100,\n        num_layers_enc=4,\n        num_layers_dec=1,\n        num_classes=60,\n    ):\n        super().__init__()\n        self.loss_metric = keras.metrics.Mean(name=\"loss\")\n        self.acc_metric = keras.metrics.Mean(name=\"edit_dist\")\n        self.num_layers_enc = num_layers_enc\n        self.num_layers_dec = num_layers_dec\n        self.target_maxlen = target_maxlen\n        self.num_classes = num_classes\n\n        self.enc_input = LandmarkEmbedding(num_hid=num_hid, maxlen=source_maxlen)\n        self.dec_input = TokenEmbedding(\n            num_vocab=num_classes, maxlen=target_maxlen, num_hid=num_hid\n        )\n\n        self.encoder = keras.Sequential(\n            [self.enc_input]\n            + [\n                TransformerEncoder(num_hid, num_head, num_feed_forward)\n                for _ in range(num_layers_enc)\n            ]\n        )\n\n        for i in range(num_layers_dec):\n            setattr(\n                self,\n                f\"dec_layer_{i}\",\n                TransformerDecoder(num_hid, num_head, num_feed_forward),\n            )\n\n        self.classifier = layers.Dense(num_classes)\n\n    def decode(self, enc_out, target, training):\n        y = self.dec_input(target)\n        for i in range(self.num_layers_dec):\n            y = getattr(self, f\"dec_layer_{i}\")(enc_out, y, training)\n        return y\n\n    def call(self, inputs, training):\n        source = inputs[0]\n        target = inputs[1]\n        x = self.encoder(source, training)\n        y = self.decode(x, target, training)\n        return self.classifier(y)\n\n    @property\n    def metrics(self):\n        return [self.loss_metric]\n\n    def train_step(self, batch):\n        \"\"\"Processes one batch inside model.fit().\"\"\"\n        source = batch[0]\n        target = batch[1]\n\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        \n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        with tf.GradientTape() as tape:\n            preds = self([source, dec_input])\n            one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n            mask = tf.math.logical_not(tf.math.equal(dec_target, pad_token_idx))\n            loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        trainable_vars = self.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n        # Computes the Levenshtein distance between sequences since the evaluation\n        # metric for this contest is the normalized total levenshtein distance.\n        edit_dist = tf.edit_distance(tf.sparse.from_dense(target), \n                                     tf.sparse.from_dense(tf.cast(tf.argmax(preds, axis=1), tf.int32)))\n        edit_dist = tf.reduce_mean(edit_dist)\n        self.acc_metric.update_state(edit_dist)\n        self.loss_metric.update_state(loss)\n        return {\"loss\": self.loss_metric.result(), \"edit_dist\": self.acc_metric.result()}\n\n    def test_step(self, batch):        \n        source = batch[0]\n        target = batch[1]\n\n        input_shape = tf.shape(target)\n        batch_size = input_shape[0]\n        \n        dec_input = target[:, :-1]\n        dec_target = target[:, 1:]\n        preds = self([source, dec_input])\n        one_hot = tf.one_hot(dec_target, depth=self.num_classes)\n        mask = tf.math.logical_not(tf.math.equal(dec_target, pad_token_idx))\n        loss = self.compiled_loss(one_hot, preds, sample_weight=mask)\n        # Computes the Levenshtein distance between sequences since the evaluation\n        # metric for this contest is the normalized total levenshtein distance.\n        edit_dist = tf.edit_distance(tf.sparse.from_dense(target), \n                                     tf.sparse.from_dense(tf.cast(tf.argmax(preds, axis=1), tf.int32)))\n        edit_dist = tf.reduce_mean(edit_dist)\n        self.acc_metric.update_state(edit_dist)\n        self.loss_metric.update_state(loss)\n        return {\"loss\": self.loss_metric.result(), \"edit_dist\": self.acc_metric.result()}\n\n    def generate(self, source, target_start_token_idx):\n        \"\"\"Performs inference over one batch of inputs using greedy decoding.\"\"\"\n        bs = tf.shape(source)[0]\n        enc = self.encoder(source, training = False)\n        dec_input = tf.ones((bs, 1), dtype=tf.int32) * target_start_token_idx\n        dec_logits = []\n        for i in range(self.target_maxlen - 1):\n            dec_out = self.decode(enc, dec_input, training = False)\n            logits = self.classifier(dec_out)\n            logits = tf.argmax(logits, axis=-1, output_type=tf.int32)\n            last_logit = logits[:, -1][..., tf.newaxis]\n            dec_logits.append(last_logit)\n            dec_input = tf.concat([dec_input, last_logit], axis=-1)\n        return dec_input","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:50.698727Z","iopub.execute_input":"2023-07-23T16:09:50.699361Z","iopub.status.idle":"2023-07-23T16:09:50.72553Z","shell.execute_reply.started":"2023-07-23T16:09:50.69933Z","shell.execute_reply":"2023-07-23T16:09:50.724707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DisplayOutputs(keras.callbacks.Callback):\n    def __init__(\n        self, batch, idx_to_token, target_start_token_idx=60, target_end_token_idx=61\n    ):\n        \"\"\"Displays a batch of outputs after every 4 epoch\n\n        Args:\n            batch: A test batch\n            idx_to_token: A List containing the vocabulary tokens corresponding to their indices\n            target_start_token_idx: A start token index in the target vocabulary\n            target_end_token_idx: An end token index in the target vocabulary\n        \"\"\"\n        self.batch = batch\n        self.target_start_token_idx = target_start_token_idx\n        self.target_end_token_idx = target_end_token_idx\n        self.idx_to_char = idx_to_token\n\n    def on_epoch_end(self, epoch, logs=None):\n        if epoch % 4 != 0:\n            return\n        source = self.batch[0]\n        target = self.batch[1].numpy()\n        bs = tf.shape(source)[0]\n        preds = self.model.generate(source, self.target_start_token_idx)\n        preds = preds.numpy()\n        for i in range(bs):\n            target_text = \"\".join([self.idx_to_char[_] for _ in target[i, :]])\n            prediction = \"\"\n            for idx in preds[i, :]:\n                prediction += self.idx_to_char[idx]\n                if idx == self.target_end_token_idx:\n                    break\n            print(f\"target:     {target_text.replace('-','')}\")\n            print(f\"prediction: {prediction}\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:50.72693Z","iopub.execute_input":"2023-07-23T16:09:50.727307Z","iopub.status.idle":"2023-07-23T16:09:50.73939Z","shell.execute_reply.started":"2023-07-23T16:09:50.727255Z","shell.execute_reply":"2023-07-23T16:09:50.738523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transformer variables are customized from original keras tutorial to suit this dataset.\n# Reference: https://www.kaggle.com/code/shlomoron/aslfr-a-simple-transformer/notebook\n\nbatch = next(iter(valid_ds))\n\n# The vocabulary to convert predicted indices into characters\nidx_to_char = list(char_to_num.keys())\ndisplay_cb = DisplayOutputs(\n    batch, idx_to_char, target_start_token_idx=char_to_num['<'], target_end_token_idx=char_to_num['>']\n)  # set the arguments as per vocabulary index for '<' and '>'\n\nmodel = Transformer(\n    num_hid=45,\n    num_head=2,\n    num_feed_forward=95,\n    source_maxlen = FRAME_LEN,\n    target_maxlen=64,\n    num_layers_enc=3,\n    num_layers_dec=1,\n    num_classes=62\n)\nloss_fn = tf.keras.losses.CategoricalCrossentropy(\n    from_logits=True, label_smoothing=0.1,\n)\n\n\noptimizer = keras.optimizers.Adam(0.001)\nmodel.compile(optimizer=optimizer, loss=loss_fn)\n\nhistory = model.fit(train_ds, validation_data=valid_ds, callbacks=[display_cb], epochs=400)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-23T16:09:50.740725Z","iopub.execute_input":"2023-07-23T16:09:50.741256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Define your Transformer model with reduced complexity and dropout\n# class TransformerRegularized(keras.Model):\n#     def __init__(self, num_hid, num_head, num_feed_forward, source_maxlen, target_maxlen, num_layers_enc, num_layers_dec, num_classes, dropout_rate=0.2):\n#         super(TransformerRegularized, self).__init__()\n#         # Define your layers with reduced complexity and dropout here\n\n#     def call(self, inputs, targets, training=False):\n#         # Implement the forward pass of your Transformer model here, applying dropout during training\n#         return x\n\n# # Define your data augmentation function\n# def data_augmentation(inputs, targets):\n#     # Apply your data augmentation logic here\n#     return inputs, targets\n\n# # Define your loss function with label smoothing\n# def loss_fn(y_true, y_pred):\n#     # Implement your loss function with label smoothing here\n#     return loss\n\n# # Define your model architecture with reduced complexity and dropout\n# model = TransformerRegularized(\n#     num_hid=50,  # Reduce the number of hidden units\n#     num_head=2,\n#     num_feed_forward=100,  # Reduce the number of feed-forward units\n#     source_maxlen=FRAME_LEN,\n#     target_maxlen=64,\n#     num_layers_enc=3,  # Reduce the number of encoder layers\n#     num_layers_dec=1,\n#     num_classes=62,\n#     dropout_rate=0.3  # Increase dropout rate\n# )\n\n# # Compile the model with your loss function\n# optimizer = keras.optimizers.Adam(learning_rate=0.0001)  # Use learning rate schedule if needed\n# model.compile(optimizer=optimizer, loss=loss_fn)\n\n# # Apply data augmentation to the training dataset\n# augmented_train_ds = train_ds.map(data_augmentation)\n\n# # Implement early stopping to prevent overfitting\n# early_stopping_cb = keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n\n# # Train your model with the augmented dataset and early stopping\n# history = model.fit(augmented_train_ds, validation_data=valid_ds, callbacks=[display_cb, early_stopping_cb], epochs=100)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.legend(['training loss', 'val_loss'])","metadata":{"execution":{"iopub.status.busy":"2023-07-23T19:32:23.25858Z","iopub.execute_input":"2023-07-23T19:32:23.258888Z","iopub.status.idle":"2023-07-23T19:32:23.668059Z","shell.execute_reply.started":"2023-07-23T19:32:23.258851Z","shell.execute_reply":"2023-07-23T19:32:23.666788Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n        self.target_start_token_idx = start_token_idx\n        self.target_end_token_idx = end_token_idx\n        # Load the feature generation and main models\n        self.model = model\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(FEATURE_COLUMNS)], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs, training=False):\n        # Preprocess Data\n        x = tf.cast(inputs, tf.float32)\n        x = x[None]\n        x = tf.cond(tf.shape(x)[1] == 0, lambda: tf.zeros((1, 1, len(FEATURE_COLUMNS))), lambda: tf.identity(x))\n        x = x[0]\n        x = pre_process(x)\n        x = x[None]\n        x = self.model.generate(x, self.target_start_token_idx)\n        x = x[0]\n        idx = tf.argmax(tf.cast(tf.equal(x, self.target_end_token_idx), tf.int32))\n        idx = tf.where(tf.math.less(idx, 1), tf.constant(2, dtype=tf.int64), idx)\n        x = x[1:idx]\n        x = tf.one_hot(x, 59)\n        return {'outputs': x}\n    \ntflitemodel_base = TFLiteModel(model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights(\"model.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]#, tf.lite.OpsSet.SELECT_TF_OPS]\ntflite_model = keras_model_converter.convert()\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \ninfargs = {\"selected_columns\" : FEATURE_COLUMNS}\n\nwith open('inference_args.json', \"w\") as json_file:\n    json.dump(infargs, json_file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip  './model.tflite' './inference_args.json'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tf.lite.Interpreter(\"model.tflite\")\n\nREQUIRED_SIGNATURE = \"serving_default\"\nREQUIRED_OUTPUT = \"outputs\"\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\nif REQUIRED_SIGNATURE not in found_signatures:\n    raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=batch[0][0])\nprediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\nprint(prediction_str)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}