{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\nimport tensorflow as tf\nimport tensorflow_addons as tfa\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit\nfrom leven import levenshtein\n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport time\nimport json\n\n# TQDM Progress Bar With Pandas Apply Function\ntqdm.pandas()\n\nprint(f'Tensorflow Version {tf.__version__}')\nprint(f'Python Version: {sys.version}')","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:40:41.39036Z","iopub.execute_input":"2023-08-31T02:40:41.390915Z","iopub.status.idle":"2023-08-31T02:40:51.119173Z","shell.execute_reply.started":"2023-08-31T02:40:41.390877Z","shell.execute_reply":"2023-08-31T02:40:51.118151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Character 2 Ordinal Encoding","metadata":{}},{"cell_type":"code","source":"# Read Character to Ordinal Encoding Mapping\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    CHAR2ORD = json.load(json_file)\n    \n# Ordinal to Character Mapping\nORD2CHAR = {j:i for i,j in CHAR2ORD.items()}\n    \n# Character to Ordinal Encoding Mapping   \ndisplay(pd.Series(CHAR2ORD).to_frame('Ordinal Encoding'))\n!pip install Levenshtein\nimport Levenshtein as lev","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:40:51.121146Z","iopub.execute_input":"2023-08-31T02:40:51.122147Z","iopub.status.idle":"2023-08-31T02:41:03.615245Z","shell.execute_reply.started":"2023-08-31T02:40:51.122112Z","shell.execute_reply":"2023-08-31T02:41:03.614078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Global Config","metadata":{}},{"cell_type":"code","source":"# If Notebook Is Run By Committing or In Interactive Mode For Development\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n# Verbose Setting during training\nVERBOSE = 1 if IS_INTERACTIVE else 2\n# Global Random Seed\nSEED = 42\n# Number of Frames to resize recording to\nN_TARGET_FRAMES = 128\n# Global debug flag, takes subset of train\nDEBUG = False\n# Number of Unique Characters To Predict + Pad Token + SOS Token + EOS Token\nN_UNIQUE_CHARACTERS0 = len(CHAR2ORD)\nN_UNIQUE_CHARACTERS = len(CHAR2ORD) + 1 + 1 + 1\nPAD_TOKEN = len(CHAR2ORD) # Padding\nSOS_TOKEN = len(CHAR2ORD) + 1 # Start Of Sentence\nEOS_TOKEN = len(CHAR2ORD) + 2 # End Of Sentence\n# Whether to use 10% of data for validation\nUSE_VAL = True\n# Batch Size\nBATCH_SIZE = 64\n# Number of Epochs to Train for\nN_EPOCHS = 2\n# Number of Warmup Epochs in Learning Rate Scheduler\nN_WARMUP_EPOCHS = 10\n# Maximum Learning Rate\nLR_MAX = 1e-3\n# Weight Decay Ratio as Ratio of Learning Rate\nWD_RATIO = 0.05\n# Length of Phrase + EOS Token\nMAX_PHRASE_LENGTH = 64##########changed from 32\n# Whether to Train The model\nTRAIN_MODEL = False\n# Whether to Load Pretrained Weights\nLOAD_WEIGHTS = True\n# Learning Rate Warmup Method [log,exp]\nWARMUP_METHOD = 'exp'","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:03.617321Z","iopub.execute_input":"2023-08-31T02:41:03.617723Z","iopub.status.idle":"2023-08-31T02:41:03.626385Z","shell.execute_reply.started":"2023-08-31T02:41:03.617682Z","shell.execute_reply":"2023-08-31T02:41:03.625354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read Train DataFrame\nif DEBUG:\n    train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv').head(5000)\nelse:\n    train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n    \n# Set Train Indexed By sqeuence_id\ntrain_sequence_id = train.set_index('sequence_id')\n\n# Number Of Train Samples\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}')\n\ndisplay(train.info())\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:03.629758Z","iopub.execute_input":"2023-08-31T02:41:03.630457Z","iopub.status.idle":"2023-08-31T02:41:03.846203Z","shell.execute_reply.started":"2023-08-31T02:41:03.630421Z","shell.execute_reply":"2023-08-31T02:41:03.845141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-fingerspelling/{path}'\n\ntrain['file_path'] = train['path'].apply(get_file_path)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:03.849222Z","iopub.execute_input":"2023-08-31T02:41:03.849536Z","iopub.status.idle":"2023-08-31T02:41:03.881907Z","shell.execute_reply.started":"2023-08-31T02:41:03.84951Z","shell.execute_reply":"2023-08-31T02:41:03.880995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example File Paths","metadata":{}},{"cell_type":"code","source":"# Unique Parquet Files\nINFERENCE_FILE_PATHS = pd.Series(\n        glob.glob('/kaggle/input/aslfr-preprocessing-dataset/train_landmark_subsets/*')\n    )\n\nprint(f'Found {len(INFERENCE_FILE_PATHS)} Inference Pickle Files')","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:03.883441Z","iopub.execute_input":"2023-08-31T02:41:03.883872Z","iopub.status.idle":"2023-08-31T02:41:03.898174Z","shell.execute_reply.started":"2023-08-31T02:41:03.883839Z","shell.execute_reply":"2023-08-31T02:41:03.897297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load X/y","metadata":{}},{"cell_type":"code","source":"# Train/Validation\nif USE_VAL:\n    # TRAIN\n    X_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/X_train.npy')\n    y_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/y_train.npy')[:,:MAX_PHRASE_LENGTH]\n    N_TRAIN_SAMPLES = len(X_train)\n    # VAL\n    X_val = np.load('/kaggle/input/aslfr-preprocessing-dataset/X_val.npy')\n    y_val = np.load('/kaggle/input/aslfr-preprocessing-dataset/y_val.npy')[:,:MAX_PHRASE_LENGTH]\n    N_VAL_SAMPLES = len(X_val)\n    # Shapes\n    print(f'X_train shape: {X_train.shape}, X_val shape: {X_val.shape}')\n# Train On All Data\nelse:\n    # TRAIN\n    X_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/X.npy')\n    y_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/y.npy')[:,:MAX_PHRASE_LENGTH]\n    N_TRAIN_SAMPLES = len(X_train)\n    print(f'X_train shape: {X_train.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:03.899918Z","iopub.execute_input":"2023-08-31T02:41:03.90026Z","iopub.status.idle":"2023-08-31T02:41:34.076034Z","shell.execute_reply.started":"2023-08-31T02:41:03.900228Z","shell.execute_reply":"2023-08-31T02:41:34.075049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Batch","metadata":{}},{"cell_type":"code","source":"# Example Batch For Debugging\nN_EXAMPLE_BATCH_SAMPLES = 1024\nN_EXAMPLE_BATCH_SAMPLES_SMALL = 32\n# Example Batch\nX_batch = {\n    'frames': np.copy(X_train[:N_EXAMPLE_BATCH_SAMPLES]),\n    'phrase': np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES]),\n#     'phrase_type': np.copy(y_phrase_type_train[:N_EXAMPLE_BATCH_SAMPLES]),\n}\ny_batch = np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES])\n# Small Example Batch\nX_batch_small = {\n    'frames': np.copy(X_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n    'phrase': np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n#     'phrase_type': np.copy(y_phrase_type_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n}\ny_batch_small = np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL])","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:34.078658Z","iopub.execute_input":"2023-08-31T02:41:34.079262Z","iopub.status.idle":"2023-08-31T02:41:34.183095Z","shell.execute_reply.started":"2023-08-31T02:41:34.079226Z","shell.execute_reply":"2023-08-31T02:41:34.182074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Parquet","metadata":{}},{"cell_type":"code","source":"# Read First Parquet File\n# example_parquet_df = pd.read_parquet(train['file_path'][0])\nexample_parquet_df = pd.read_parquet(INFERENCE_FILE_PATHS[0])\n\n# Each parquet file contains 1000 recordings\nprint(f'# Unique Recording: {example_parquet_df.index.nunique()}')\n# Display DataFrame layout\ndisplay(example_parquet_df.head())","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:34.184766Z","iopub.execute_input":"2023-08-31T02:41:34.185593Z","iopub.status.idle":"2023-08-31T02:41:35.45765Z","shell.execute_reply.started":"2023-08-31T02:41:34.185553Z","shell.execute_reply":"2023-08-31T02:41:35.45671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Indices","metadata":{}},{"cell_type":"code","source":"# Get indices in original dataframe\ndef get_idxs(df, words_pos, words_neg=[], ret_names=True, idxs_pos=None):\n    idxs = []\n    names = []\n    for w in words_pos:\n        for col_idx, col in enumerate(example_parquet_df.columns):\n            # Exclude Non Landmark Columns\n            if col in ['frame']:\n                continue\n                \n            col_idx = int(col.split('_')[-1])\n            # Check if column name contains all words\n            if (w in col) and (idxs_pos is None or col_idx in idxs_pos) and all([w not in col for w in words_neg]):\n                idxs.append(col_idx)\n                names.append(col)\n    # Convert to Numpy arrays\n    idxs = np.array(idxs)\n    names = np.array(names)\n    # Returns either both column indices and names\n    if ret_names:\n        return idxs, names\n    # Or only columns indices\n    else:\n        return idxs","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:35.462233Z","iopub.execute_input":"2023-08-31T02:41:35.462801Z","iopub.status.idle":"2023-08-31T02:41:35.472773Z","shell.execute_reply.started":"2023-08-31T02:41:35.462773Z","shell.execute_reply":"2023-08-31T02:41:35.47176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lips Landmark Face Ids\nLIPS_LANDMARK_IDXS = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\n\n# Landmark Indices for Left/Right hand without z axis in raw data\nLEFT_HAND_IDXS0, LEFT_HAND_NAMES0 = get_idxs(example_parquet_df, ['left_hand'], ['z'])\nRIGHT_HAND_IDXS0, RIGHT_HAND_NAMES0 = get_idxs(example_parquet_df, ['right_hand'], ['z'])\nLIPS_IDXS0, LIPS_NAMES0 = get_idxs(example_parquet_df, ['face'], ['z'], idxs_pos=LIPS_LANDMARK_IDXS)\nCOLUMNS0 = np.concatenate((LEFT_HAND_NAMES0, RIGHT_HAND_NAMES0, LIPS_NAMES0))\nN_COLS0 = len(COLUMNS0)\n# Only X/Y axes are used\nN_DIMS0 = 2\n\nprint(f'N_COLS0: {N_COLS0}')","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:35.474058Z","iopub.execute_input":"2023-08-31T02:41:35.474997Z","iopub.status.idle":"2023-08-31T02:41:35.48679Z","shell.execute_reply.started":"2023-08-31T02:41:35.47496Z","shell.execute_reply":"2023-08-31T02:41:35.485442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Landmark Indices in subset of dataframe with only COLUMNS selected\nLEFT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, LEFT_HAND_NAMES0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, RIGHT_HAND_NAMES0)).squeeze()\nLIPS_IDXS = np.argwhere(np.isin(COLUMNS0, LIPS_NAMES0)).squeeze()\nHAND_IDXS = np.concatenate((LEFT_HAND_IDXS, RIGHT_HAND_IDXS), axis=0)\nN_COLS = N_COLS0\n# Only X/Y axes are used\nN_DIMS = 2\n\nprint(f'N_COLS: {N_COLS}')","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:35.488215Z","iopub.execute_input":"2023-08-31T02:41:35.489275Z","iopub.status.idle":"2023-08-31T02:41:35.499366Z","shell.execute_reply.started":"2023-08-31T02:41:35.489242Z","shell.execute_reply":"2023-08-31T02:41:35.498208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Indices in processed data by axes with only dominant hand\nHAND_X_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'x' in name]\n    ).squeeze()\nHAND_Y_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'y' in name]\n    ).squeeze()\n# Names in processed data by axes\nHAND_X_NAMES = LEFT_HAND_NAMES0[HAND_X_IDXS]\nHAND_Y_NAMES = LEFT_HAND_NAMES0[HAND_Y_IDXS]","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:35.501351Z","iopub.execute_input":"2023-08-31T02:41:35.501795Z","iopub.status.idle":"2023-08-31T02:41:35.509318Z","shell.execute_reply.started":"2023-08-31T02:41:35.501761Z","shell.execute_reply":"2023-08-31T02:41:35.508125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mean/STD Loading","metadata":{}},{"cell_type":"code","source":"# Mean/Standard Deviations of data used for normalizing\nMEANS = np.load('/kaggle/input/aslfr-preprocessing-dataset/MEANS.npy').reshape(-1)\nSTDS = np.load('/kaggle/input/aslfr-preprocessing-dataset/STDS.npy').reshape(-1)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:35.511233Z","iopub.execute_input":"2023-08-31T02:41:35.511568Z","iopub.status.idle":"2023-08-31T02:41:35.524446Z","shell.execute_reply.started":"2023-08-31T02:41:35.511538Z","shell.execute_reply":"2023-08-31T02:41:35.523611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorflow Preprocessing Layer","metadata":{}},{"cell_type":"code","source":"\"\"\"\n    Tensorflow layer to process data in TFLite\n    Data needs to be processed in the model itself, so we can not use Python\n\"\"\" \nclass PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n        self.normalisation_correction = tf.constant(\n                    # Add 0.50 to x coordinates of left hand (original right hand) and substract 0.50 of right hand (original left hand)\n                     [0.50 if 'x' in name else 0.00 for name in LEFT_HAND_NAMES0],\n                dtype=tf.float32,\n            )\n    \n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,N_COLS0], dtype=tf.float32),),\n    )\n    def call(self, data0, resize=True):\n        # Fill NaN Values With 0\n        data = tf.where(tf.math.is_nan(data0), 0.0, data0)\n        \n        # Hacky\n        data = data[None]\n        \n        # Empty Hand Frame Filtering\n        hands = tf.slice(data, [0,0,0], [-1, -1, 84])\n        hands = tf.abs(hands)\n        mask = tf.reduce_sum(hands, axis=2)\n        mask = tf.not_equal(mask, 0)\n        data = data[mask][None]\n        \n        # Pad Zeros\n        N_FRAMES = len(data[0])\n        if N_FRAMES < N_TARGET_FRAMES:\n            data = tf.concat((\n                data,\n                tf.zeros([1,N_TARGET_FRAMES-N_FRAMES,N_COLS], dtype=tf.float32)\n            ), axis=1)\n        # Downsample\n        data = tf.image.resize(\n            data,\n            [1, N_TARGET_FRAMES],\n            method=tf.image.ResizeMethod.BILINEAR,\n        )\n        \n        # Squeeze Batch Dimension\n        data = tf.squeeze(data, axis=[0])\n        \n        return data\n    \npreprocess_layer = PreprocessLayer()","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:35.525799Z","iopub.execute_input":"2023-08-31T02:41:35.526136Z","iopub.status.idle":"2023-08-31T02:41:38.426271Z","shell.execute_reply.started":"2023-08-31T02:41:35.526104Z","shell.execute_reply":"2023-08-31T02:41:38.425113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function To Test Preprocessing Layer\ndef test_preprocess_layer():\n    demo_sequence_id = example_parquet_df.index.unique()[15]\n    demo_raw_data = example_parquet_df.loc[demo_sequence_id, COLUMNS0]\n    data = preprocess_layer(demo_raw_data)\n\n    print(f'demo_raw_data shape: {demo_raw_data.shape}')\n    print(f'data shape: {data.shape}')\n    \n    return data\n    \nif IS_INTERACTIVE:\n    data = test_preprocess_layer()","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.427932Z","iopub.execute_input":"2023-08-31T02:41:38.428285Z","iopub.status.idle":"2023-08-31T02:41:38.818046Z","shell.execute_reply.started":"2023-08-31T02:41:38.428251Z","shell.execute_reply":"2023-08-31T02:41:38.817065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Dataset","metadata":{}},{"cell_type":"code","source":"# Train Dataset Iterator\ndef get_train_dataset(X, y, batch_size=BATCH_SIZE):\n    sample_idxs = np.arange(len(X))\n    while True:\n        # Get random indices\n        random_sample_idxs = np.random.choice(sample_idxs, batch_size)\n        \n        inputs = {\n            'frames': X[random_sample_idxs],\n            'phrase': y[random_sample_idxs],\n        }\n        outputs = y[random_sample_idxs]\n        \n        yield inputs, outputs","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.819323Z","iopub.execute_input":"2023-08-31T02:41:38.820302Z","iopub.status.idle":"2023-08-31T02:41:38.827745Z","shell.execute_reply.started":"2023-08-31T02:41:38.820266Z","shell.execute_reply":"2023-08-31T02:41:38.825614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Dataset\ntrain_dataset = get_train_dataset(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.829092Z","iopub.execute_input":"2023-08-31T02:41:38.829732Z","iopub.status.idle":"2023-08-31T02:41:38.837146Z","shell.execute_reply.started":"2023-08-31T02:41:38.829698Z","shell.execute_reply":"2023-08-31T02:41:38.835883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Steps Per Epoch\nTRAIN_STEPS_PER_EPOCH = math.ceil(N_TRAIN_SAMPLES / BATCH_SIZE)\nprint(f'TRAIN_STEPS_PER_EPOCH: {TRAIN_STEPS_PER_EPOCH}')","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.838503Z","iopub.execute_input":"2023-08-31T02:41:38.839018Z","iopub.status.idle":"2023-08-31T02:41:38.84814Z","shell.execute_reply.started":"2023-08-31T02:41:38.838986Z","shell.execute_reply":"2023-08-31T02:41:38.84694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation Dataset","metadata":{}},{"cell_type":"code","source":"# Validation Set\ndef get_val_dataset(X, y, batch_size=BATCH_SIZE):\n    offsets = np.arange(0, len(X), batch_size)\n    while True:\n        # Iterate over whole validation set\n        for offset in offsets:\n            inputs = {\n                'frames': X[offset:offset+batch_size],\n                'phrase': y[offset:offset+batch_size],\n            }\n            outputs = y[offset:offset+batch_size]\n\n            yield inputs, outputs","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.849614Z","iopub.execute_input":"2023-08-31T02:41:38.850223Z","iopub.status.idle":"2023-08-31T02:41:38.857392Z","shell.execute_reply.started":"2023-08-31T02:41:38.85019Z","shell.execute_reply":"2023-08-31T02:41:38.85651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Jumbled dataset\ndef get_dummy_dataset(X,y, batch_size=BATCH_SIZE):\n    offsets = np.arange(0, len(X), batch_size)\n    y_t= y[:]\n    np.random.shuffle(y_t)\n    while True:\n        for offset in offsets:\n            inputs = {\n                'frames': X[offset:offset+batch_size],\n                'phrase': y_t[offset:offset+batch_size]\n            }\n            outputs = y_t[offset:offset+batch_size]\n\n            yield inputs, outputs\ndeb_dataset= get_dummy_dataset(X_val,y_val)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.858856Z","iopub.execute_input":"2023-08-31T02:41:38.859204Z","iopub.status.idle":"2023-08-31T02:41:38.86696Z","shell.execute_reply.started":"2023-08-31T02:41:38.859171Z","shell.execute_reply":"2023-08-31T02:41:38.86597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation Dataset\nif USE_VAL:\n    val_dataset = get_val_dataset(X_val, y_val)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.868255Z","iopub.execute_input":"2023-08-31T02:41:38.868936Z","iopub.status.idle":"2023-08-31T02:41:38.875826Z","shell.execute_reply.started":"2023-08-31T02:41:38.868902Z","shell.execute_reply":"2023-08-31T02:41:38.874849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    N_VAL_STEPS_PER_EPOCH = math.ceil(N_VAL_SAMPLES / BATCH_SIZE)\n    print(f'N_VAL_STEPS_PER_EPOCH: {N_VAL_STEPS_PER_EPOCH}')","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.878943Z","iopub.execute_input":"2023-08-31T02:41:38.87921Z","iopub.status.idle":"2023-08-31T02:41:38.888475Z","shell.execute_reply.started":"2023-08-31T02:41:38.879187Z","shell.execute_reply":"2023-08-31T02:41:38.887518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Config","metadata":{}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# final embedding and transformer embedding size\nUNITS_ENCODER = 384\nUNITS_DECODER = 256\n\n# Transformer\nNUM_BLOCKS_ENCODER = 4\nNUM_BLOCKS_DECODER = 2\nNUM_HEADS = 4\nMLP_RATIO = 2\n\n# Dropout\nEMBEDDING_DROPOUT = 0.00\nMLP_DROPOUT_RATIO = 0.30\nMHA_DROPOUT_RATIO = 0.20\nCLASSIFIER_DROPOUT_RATIO = 0.10\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.890159Z","iopub.execute_input":"2023-08-31T02:41:38.890931Z","iopub.status.idle":"2023-08-31T02:41:38.897953Z","shell.execute_reply.started":"2023-08-31T02:41:38.890897Z","shell.execute_reply":"2023-08-31T02:41:38.896953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Embedding","metadata":{}},{"cell_type":"code","source":"# Embeds a landmark using fully connected layers\nclass LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n        self.supports_masking = True\n        \n    def build(self, input_shape):\n        # Embedding for missing landmark in frame, initizlied with zeros\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # Embedding\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.dense(x),\n            )","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.899399Z","iopub.execute_input":"2023-08-31T02:41:38.900078Z","iopub.status.idle":"2023-08-31T02:41:38.909599Z","shell.execute_reply.started":"2023-08-31T02:41:38.900044Z","shell.execute_reply":"2023-08-31T02:41:38.908883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Embedding","metadata":{}},{"cell_type":"code","source":"# Creates embedding for each frame\nclass Embedding(tf.keras.Model):\n    def __init__(self):\n        super(Embedding, self).__init__()\n        self.supports_masking = True\n    \n    def build(self, input_shape):\n        # Positional embedding for each frame index\n        self.positional_embedding = tf.Variable(\n            initial_value=tf.zeros([N_TARGET_FRAMES, UNITS_ENCODER], dtype=tf.float32),\n            trainable=True,\n            name='embedding_positional_encoder',\n        )\n        # Embedding layer for Landmarks\n        self.dominant_hand_embedding = LandmarkEmbedding(UNITS_ENCODER, 'dominant_hand')\n\n    def call(self, x, training=False):\n        # Normalize\n        x = tf.where(\n                tf.math.equal(x, 0.0),\n                0.0,\n                (x - MEANS) / STDS,\n            )\n        # Dominant Hand\n        x = self.dominant_hand_embedding(x)\n        # Add Positional Encoding\n        x = x + self.positional_embedding\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.911085Z","iopub.execute_input":"2023-08-31T02:41:38.911753Z","iopub.status.idle":"2023-08-31T02:41:38.922572Z","shell.execute_reply.started":"2023-08-31T02:41:38.911719Z","shell.execute_reply":"2023-08-31T02:41:38.921967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer","metadata":{}},{"cell_type":"code","source":"# based on: https://stackoverflow.com/questions/67342988/verifying-the-implementation-of-multihead-attention-in-transformer\n# replaced softmax with softmax layer to support masked softmax\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self, d_model, n_heads, dropout, d_out=None):\n        super(MultiHeadAttention,self).__init__()\n        # Number of Units in Model\n        self.d_model = d_model\n        # Number of Attention Heads\n        self.n_heads = n_heads\n        # Number of Units in Intermediate Layers\n        self.depth = d_model // 2\n        # Scaling Factor Of Values\n        self.scale = 1.0 / tf.math.sqrt(tf.cast(self.depth, tf.float32))\n        # Learnable Projections to Depth\n        self.wq = self.fused_mha(self.depth)\n        self.wk = self.fused_mha(self.depth)\n        self.wv = self.fused_mha(self.depth)\n        # Output Projection\n        self.wo = tf.keras.layers.Dense(d_model if d_out is None else d_out, use_bias=False)\n        # Softmax Activation Which Supports Masking\n        self.softmax = tf.keras.layers.Softmax()\n        # Reshaping Of Multiple Attention heads to Single Value\n        self.reshape = tf.keras.Sequential([\n            # [attention heads, number of frames, d_model] → [number of frames, n_heads, d_model // n_heads]\n            tf.keras.layers.Permute([2, 1, 3]),\n            # [number of frames, attention heads, d_model] → [number of frames, d_model]\n            tf.keras.layers.Reshape([N_TARGET_FRAMES, self.depth]),\n        ])\n        # Output Dropout\n        self.do = tf.keras.layers.Dropout(dropout)\n        self.supports_masking = True\n        \n    # Single dense layer for all attention heads\n    def fused_mha(self, dim):\n        return tf.keras.Sequential([\n            # Single dense layer\n            tf.keras.layers.Dense(dim, use_bias=False),\n            # Reshape to [number of frames, number of attention head, depth]\n            tf.keras.layers.Reshape([N_TARGET_FRAMES, self.n_heads, dim // self.n_heads]),\n            # Permutate to [number of attention heads, number of frames, depth]\n            tf.keras.layers.Permute([2, 1, 3]),\n        ])\n        \n    def call(self, q, k, v, attention_mask=None, training=False):\n        # Projections to attention heads\n        Q = self.wq(q)\n        K = self.wk(k)\n        V = self.wv(v)\n        # Matrix multiply QxK to acquire attention scores\n        x = tf.matmul(Q, K, transpose_b=True) * self.scale\n        # Softmax attention scores and Multiply with Values\n        x = self.softmax(x, mask=attention_mask) @ V\n        # Reshape to flatten attention heads\n        x = self.reshape(x)\n        # Output projection\n        x = self.wo(x)\n        # Dropout\n        x = self.do(x, training=training)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.924115Z","iopub.execute_input":"2023-08-31T02:41:38.924834Z","iopub.status.idle":"2023-08-31T02:41:38.940045Z","shell.execute_reply.started":"2023-08-31T02:41:38.924799Z","shell.execute_reply":"2023-08-31T02:41:38.939083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encoder\n\n[source](https://keras.io/examples/nlp/neural_machine_translation_with_transformer/)","metadata":{}},{"cell_type":"code","source":"# Encoder based on multiple transformer blocks\nclass Encoder(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Encoder, self).__init__(name='encoder')\n        self.num_blocks = num_blocks\n        self.supports_masking = True\n    \n    def build(self, input_shape):\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS_ENCODER, NUM_HEADS, MHA_DROPOUT_RATIO))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS_ENCODER * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS_ENCODER, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),\n            ]))\n            # Optional Projection to Decoder Dimension\n            if UNITS_ENCODER != UNITS_DECODER:\n                self.dense_out = tf.keras.layers.Dense(UNITS_DECODER, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False)\n                self.apply_dense_out = True\n            else:\n                self.apply_dense_out = False\n                \n    def get_attention_mask(self, x_inp):\n        # Attention Mask\n        attention_mask = tf.math.count_nonzero(x_inp, axis=[2], keepdims=True, dtype=tf.int32)\n        attention_mask = tf.math.count_nonzero(attention_mask, axis=[2], keepdims=False)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        return attention_mask\n        \n    def call(self, x, x_inp, training=False):\n        # Attention mask to ignore missing frames\n        attention_mask = self.get_attention_mask(x_inp)\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x = ln_1(x + mha(x, x, x, attention_mask=attention_mask))\n            x = ln_2(x + mlp(x))\n            \n        # Optional Projection to Decoder Dimension\n        if self.apply_dense_out:\n            x = self.dense_out(x)\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.947175Z","iopub.execute_input":"2023-08-31T02:41:38.947807Z","iopub.status.idle":"2023-08-31T02:41:38.962359Z","shell.execute_reply.started":"2023-08-31T02:41:38.94778Z","shell.execute_reply":"2023-08-31T02:41:38.961444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Decoder","metadata":{}},{"cell_type":"code","source":"# Decoder based on multiple transformer blocks\nclass Decoder(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Decoder, self).__init__(name='decoder')\n        self.num_blocks = num_blocks\n        self.supports_masking = True\n    \n    def build(self, input_shape):\n        # Causal Mask Batch Size 1\n        self.causal_mask = self.get_causal_attention_mask()\n        # Positional Embedding, initialized with zeros\n        self.positional_embedding = tf.Variable(\n            initial_value=tf.zeros([N_TARGET_FRAMES, UNITS_DECODER], dtype=tf.float32),\n            trainable=True,\n            name='embedding_positional_encoder',\n        )\n        # Character Embedding\n        self.char_emb = tf.keras.layers.Embedding(N_UNIQUE_CHARACTERS, UNITS_DECODER, embeddings_initializer=INIT_ZEROS)\n        # Positional Encoder MHA\n        self.pos_emb_mha = MultiHeadAttention(UNITS_DECODER, NUM_HEADS, MHA_DROPOUT_RATIO)\n        self.pos_emb_ln = tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS)\n        # First Layer Normalisation\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS_DECODER, NUM_HEADS, MHA_DROPOUT_RATIO))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS_DECODER * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS_DECODER, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),\n            ]))\n            \n    def get_causal_attention_mask(self):\n        i = tf.range(N_TARGET_FRAMES)[:, tf.newaxis]\n        j = tf.range(N_TARGET_FRAMES)\n        mask = tf.cast(i >= j, dtype=tf.int32)\n        mask = tf.reshape(mask, (1, N_TARGET_FRAMES, N_TARGET_FRAMES))\n        mult = tf.concat(\n            [tf.expand_dims(1, -1), tf.constant([1, 1], dtype=tf.int32)],\n            axis=0,\n        )\n        mask = tf.tile(mask, mult)\n        mask = tf.cast(mask, tf.float32)\n        return mask\n    \n    def get_attention_mask(self, x_inp):\n        # Attention Mask\n        attention_mask = tf.math.count_nonzero(x_inp, axis=[2], keepdims=True, dtype=tf.int32)\n        attention_mask = tf.math.count_nonzero(attention_mask, axis=[2], keepdims=False)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        return attention_mask\n        \n    def call(self, encoder_outputs, phrase, x_inp, training=False):\n        # Batch Size\n        B = tf.shape(encoder_outputs)[0]\n        # Cast to INT32\n        phrase = tf.cast(phrase, tf.int32)\n        # Prepend SOS Token\n        phrase = tf.pad(phrase, [[0,0], [1,0]], constant_values=SOS_TOKEN, name='prepend_sos_token')\n        # Pad With PAD Token\n        phrase = tf.pad(phrase, [[0,0], [0,N_TARGET_FRAMES-MAX_PHRASE_LENGTH-1]], constant_values=PAD_TOKEN, name='append_pad_token')\n        # Positional Embedding\n        x = self.positional_embedding + self.char_emb(phrase)\n        # Causal Attention\n        x = self.pos_emb_ln(x+self.pos_emb_mha(x, x, x, attention_mask=self.causal_mask))\n        # Attention mask to ignore missing frames\n        attention_mask = self.get_attention_mask(x_inp)\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x = ln_1(x + mha(x, encoder_outputs, encoder_outputs, attention_mask=attention_mask))\n            x = ln_2(x + mlp(x))\n        # Slice 31 Characters\n        x = tf.slice(x, [0, 0, 0], [-1, MAX_PHRASE_LENGTH, -1])\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.965944Z","iopub.execute_input":"2023-08-31T02:41:38.966217Z","iopub.status.idle":"2023-08-31T02:41:38.988605Z","shell.execute_reply.started":"2023-08-31T02:41:38.966194Z","shell.execute_reply":"2023-08-31T02:41:38.987583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def CTCLoss(labels, logits):\n    label_length = tf.reduce_sum(tf.cast(labels != PAD_TOKEN, tf.int32), axis=-1)\n    logit_length = tf.ones(tf.shape(logits)[0], dtype=tf.int32) * tf.shape(logits)[1]\n    loss = tf.nn.ctc_loss(\n            labels=labels,\n            logits=logits,\n            label_length=label_length,\n            logit_length=logit_length,\n            blank_index=PAD_TOKEN,\n            logits_time_major=False\n        )\n    \n\n    loss = tf.reduce_mean(loss)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:38.990527Z","iopub.execute_input":"2023-08-31T02:41:38.991283Z","iopub.status.idle":"2023-08-31T02:41:39.000986Z","shell.execute_reply.started":"2023-08-31T02:41:38.99125Z","shell.execute_reply":"2023-08-31T02:41:38.999955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    # Inputs\n    frames_inp = tf.keras.layers.Input([N_TARGET_FRAMES, N_COLS], dtype=tf.float32, name='frames')\n    phrase_inp = tf.keras.layers.Input([MAX_PHRASE_LENGTH], dtype=tf.int32, name='phrase')\n    # Frames\n    x = frames_inp\n\n    # Masking\n    x = tf.keras.layers.Masking(mask_value=0.0, input_shape=(N_TARGET_FRAMES, N_COLS))(x)\n    \n    # Embedding\n    x = Embedding()(x)\n    \n    # Encoder Transformer Blocks\n    x = Encoder(NUM_BLOCKS_ENCODER)(x, frames_inp)\n    \n    # Decoder\n    x = Decoder(NUM_BLOCKS_DECODER)(x, phrase_inp, frames_inp)\n\n    # Classifier\n    x = tf.keras.Sequential([\n        # Dropout\n        tf.keras.layers.Dropout(CLASSIFIER_DROPOUT_RATIO),\n        # Output Neurons\n        tf.keras.layers.Dense(N_UNIQUE_CHARACTERS, activation=tf.keras.activations.linear, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),\n    ], name='classifier')(x)\n    \n    outputs = x\n    \n    # Create Tensorflow Model\n    model = tf.keras.models.Model(inputs=[frames_inp, phrase_inp], outputs=outputs)\n    \n    # Categorical Crossentropy Loss With Label Smoothing\n    loss = CTCLoss\n    \n    # Adam Optimizer\n    optimizer = tfa.optimizers.RectifiedAdam(sma_threshold=4)\n    optimizer = tfa.optimizers.Lookahead(optimizer, sync_period=5)\n\n    # TopK Metrics\n#     metrics = [\n#         TopKAccuracy(1),\n#         TopKAccuracy(5),\n#     ]\n    \n    model.compile(\n        loss=loss,\n        optimizer=optimizer,\n#         metrics=metrics,\n      #  loss_weights=loss_weights,\n    )\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:39.002376Z","iopub.execute_input":"2023-08-31T02:41:39.00288Z","iopub.status.idle":"2023-08-31T02:41:39.01472Z","shell.execute_reply.started":"2023-08-31T02:41:39.002848Z","shell.execute_reply":"2023-08-31T02:41:39.013802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = get_model()","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:39.017884Z","iopub.execute_input":"2023-08-31T02:41:39.01817Z","iopub.status.idle":"2023-08-31T02:41:41.968207Z","shell.execute_reply.started":"2023-08-31T02:41:39.018146Z","shell.execute_reply":"2023-08-31T02:41:41.966804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:41.969864Z","iopub.execute_input":"2023-08-31T02:41:41.970201Z","iopub.status.idle":"2023-08-31T02:41:42.121005Z","shell.execute_reply.started":"2023-08-31T02:41:41.970169Z","shell.execute_reply":"2023-08-31T02:41:42.120287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Model Architecture\ntf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True, show_layer_activations=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:41:42.121995Z","iopub.execute_input":"2023-08-31T02:41:42.122329Z","iopub.status.idle":"2023-08-31T02:41:42.408423Z","shell.execute_reply.started":"2023-08-31T02:41:42.122296Z","shell.execute_reply":"2023-08-31T02:41:42.407523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_val_lev(dataset):\n    preds = []\n    targets = []\n    scores = []\n    t=0\n    for d,p in dataset:\n        t+=1\n        if t==N_VAL_STEPS_PER_EPOCH:\n            break\n        preds_batch = model.predict(d, verbose = 0)\n        targets_batch = p\n        for pred_idx in range(len(preds_batch)):\n            preds.append(\"\".join([ORD2CHAR.get(s, \"\") for s in decode_phrase(preds_batch[pred_idx]).numpy()]))\n            targets.append(\"\".join([ORD2CHAR.get(s, \"\") for s in targets_batch[pred_idx]]))\n        print('{0:30s}  {1:30s}'.format(preds[-1],targets[-1]))\n    N= [len(phrase) for phrase in targets]\n    lev_dist = [lev.distance(preds[i], targets[i]) for i in range(len(targets))]\n    print('\\nLev distance: '+str((np.sum(N) - np.sum(lev_dist))/np.sum(N)))","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:42:42.760796Z","iopub.execute_input":"2023-08-31T02:42:42.761192Z","iopub.status.idle":"2023-08-31T02:42:42.771066Z","shell.execute_reply.started":"2023-08-31T02:42:42.761163Z","shell.execute_reply":"2023-08-31T02:42:42.769877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function(jit_compile=True)\ndef decode_phrase(pred):\n    x = tf.argmax(pred, axis=1)\n    diff = tf.not_equal(x[:-1], x[1:])\n    adjacent_indices = tf.where(diff)[:, 0]\n    x = tf.gather(x, adjacent_indices)\n    mask = tf.not_equal(x,PAD_TOKEN)\n    x = tf.boolean_mask(x, mask, axis=0)\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:42:46.725118Z","iopub.execute_input":"2023-08-31T02:42:46.725487Z","iopub.status.idle":"2023-08-31T02:42:46.732285Z","shell.execute_reply.started":"2023-08-31T02:42:46.72545Z","shell.execute_reply":"2023-08-31T02:42:46.730963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights('/kaggle/input/97vald/model_epoch_44(2).h5')","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:42:48.004422Z","iopub.execute_input":"2023-08-31T02:42:48.0048Z","iopub.status.idle":"2023-08-31T02:42:48.599924Z","shell.execute_reply.started":"2023-08-31T02:42:48.004769Z","shell.execute_reply":"2023-08-31T02:42:48.598916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calculate_val_lev(val_dataset)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:42:48.621153Z","iopub.execute_input":"2023-08-31T02:42:48.622066Z","iopub.status.idle":"2023-08-31T02:43:13.374225Z","shell.execute_reply.started":"2023-08-31T02:42:48.62203Z","shell.execute_reply":"2023-08-31T02:43:13.37311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calculate_val_lev(deb_dataset)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:43:13.376303Z","iopub.execute_input":"2023-08-31T02:43:13.376671Z","iopub.status.idle":"2023-08-31T02:43:33.011987Z","shell.execute_reply.started":"2023-08-31T02:43:13.376616Z","shell.execute_reply":"2023-08-31T02:43:33.010697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(val_dataset,batch_size=BATCH_SIZE,steps=N_VAL_STEPS_PER_EPOCH)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:43:33.013613Z","iopub.execute_input":"2023-08-31T02:43:33.013992Z","iopub.status.idle":"2023-08-31T02:43:56.094838Z","shell.execute_reply.started":"2023-08-31T02:43:33.013949Z","shell.execute_reply":"2023-08-31T02:43:56.093585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## dummy dataset (jumbled targets)\nmodel.evaluate(deb_dataset,batch_size=BATCH_SIZE,steps=N_VAL_STEPS_PER_EPOCH)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T02:43:56.096648Z","iopub.execute_input":"2023-08-31T02:43:56.096945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}