{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":52950,"databundleVersionId":5973250,"sourceType":"competition"},{"sourceId":5835808,"sourceType":"datasetVersion","datasetId":3354626},{"sourceId":5939194,"sourceType":"datasetVersion","datasetId":3282038}],"dockerImageVersionId":30498,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\nimport tensorflow as tf\nimport tensorflow_addons as tfa\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit\nfrom leven import levenshtein\n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport time\nimport json\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n# TQDM Progress Bar With Pandas Apply Function\ntqdm.pandas()\n\nprint(f'Tensorflow Version {tf.__version__}')\nprint(f'Python Version: {sys.version}')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:24.240392Z","iopub.execute_input":"2024-04-01T12:35:24.241147Z","iopub.status.idle":"2024-04-01T12:35:24.250374Z","shell.execute_reply.started":"2024-04-01T12:35:24.241095Z","shell.execute_reply":"2024-04-01T12:35:24.249473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Character 2 Ordinal Encoding","metadata":{}},{"cell_type":"code","source":"# Read Character to Ordinal Encoding Mapping\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    CHAR2ORD = json.load(json_file)\n    \n# Ordinal to Character Mapping\nORD2CHAR = {j:i for i,j in CHAR2ORD.items()}\n    \n# Character to Ordinal Encoding Mapping   \ndisplay(pd.Series(CHAR2ORD).to_frame('Ordinal Encoding'))","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:24.252056Z","iopub.execute_input":"2024-04-01T12:35:24.252317Z","iopub.status.idle":"2024-04-01T12:35:24.276045Z","shell.execute_reply.started":"2024-04-01T12:35:24.252294Z","shell.execute_reply":"2024-04-01T12:35:24.275174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Global Config","metadata":{}},{"cell_type":"code","source":"# If Notebook Is Run By Committing or In Interactive Mode For Development\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n# Verbose Setting during training\nVERBOSE = 1 if IS_INTERACTIVE else 2\n# Global Random Seed\nSEED = 42\n# Number of Frames to resize recording to\nN_TARGET_FRAMES = 128\n\n# Number of Unique Characters To Predict + Pad Token + SOS Token + EOS Token\nN_UNIQUE_CHARACTERS0 = len(CHAR2ORD)\nN_UNIQUE_CHARACTERS = len(CHAR2ORD) + 1 + 1 + 1\nPAD_TOKEN = len(CHAR2ORD) # Padding\nSOS_TOKEN = len(CHAR2ORD) + 1 # Start Of Sentence\nEOS_TOKEN = len(CHAR2ORD) + 2 # End Of Sentence\n# Whether to use 10% of data for validation\nUSE_VAL = False\n# Batch Size\nBATCH_SIZE = 64\n# Number of Epochs to Train for\nN_EPOCHS = 2 if IS_INTERACTIVE else 100\n# Number of Warmup Epochs in Learning Rate Scheduler\nN_WARMUP_EPOCHS = 10\n# Maximum Learning Rate\nLR_MAX = 1e-3\n# Weight Decay Ratio as Ratio of Learning Rate\nWD_RATIO = 0.05\n# Length of Phrase + EOS Token\nMAX_PHRASE_LENGTH = 31 + 1\n# Whether to Train The model\nTRAIN_MODEL = True\n# Whether to Load Pretrained Weights\nLOAD_WEIGHTS = False\n# Learning Rate Warmup Method [log,exp]\nWARMUP_METHOD = 'exp'","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:24.277545Z","iopub.execute_input":"2024-04-01T12:35:24.277814Z","iopub.status.idle":"2024-04-01T12:35:24.285587Z","shell.execute_reply.started":"2024-04-01T12:35:24.277791Z","shell.execute_reply":"2024-04-01T12:35:24.284568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Config","metadata":{}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:24.28688Z","iopub.execute_input":"2024-04-01T12:35:24.287252Z","iopub.status.idle":"2024-04-01T12:35:24.297443Z","shell.execute_reply.started":"2024-04-01T12:35:24.287202Z","shell.execute_reply":"2024-04-01T12:35:24.296579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# Read Train DataFrame\ntrain = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n    \n# Set Train Indexed By sqeuence_id\ntrain_sequence_id = train.set_index('sequence_id')\n\n# Number Of Train Samples\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}')\n\ndisplay(train.info())\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:24.300125Z","iopub.execute_input":"2024-04-01T12:35:24.300528Z","iopub.status.idle":"2024-04-01T12:35:24.473423Z","shell.execute_reply.started":"2024-04-01T12:35:24.300471Z","shell.execute_reply":"2024-04-01T12:35:24.472528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# File Path","metadata":{}},{"cell_type":"code","source":"# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-fingerspelling/{path}'\n\ntrain['file_path'] = train['path'].apply(get_file_path)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:24.474748Z","iopub.execute_input":"2024-04-01T12:35:24.475134Z","iopub.status.idle":"2024-04-01T12:35:24.504103Z","shell.execute_reply.started":"2024-04-01T12:35:24.475101Z","shell.execute_reply":"2024-04-01T12:35:24.503262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example File Paths","metadata":{}},{"cell_type":"code","source":"# Unique Parquet Files\nINFERENCE_FILE_PATHS = pd.Series(\n        glob.glob('/kaggle/input/aslfr-preprocessing-dataset/train_landmark_subsets/*')\n    )\n\nprint(f'Found {len(INFERENCE_FILE_PATHS)} Inference Pickle Files')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:24.505594Z","iopub.execute_input":"2024-04-01T12:35:24.506001Z","iopub.status.idle":"2024-04-01T12:35:24.513912Z","shell.execute_reply.started":"2024-04-01T12:35:24.505961Z","shell.execute_reply":"2024-04-01T12:35:24.512916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load X/y","metadata":{}},{"cell_type":"code","source":"# Train/Validation\nif USE_VAL:\n    # TRAIN\n    X_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/X_train.npy')\n    y_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/y_train.npy')[:,:MAX_PHRASE_LENGTH]\n    N_TRAIN_SAMPLES = len(X_train)\n    # VAL\n    X_val = np.load('/kaggle/input/aslfr-preprocessing-dataset/X_val.npy')\n    y_val = np.load('/kaggle/input/aslfr-preprocessing-dataset/y_val.npy')[:,:MAX_PHRASE_LENGTH]\n    N_VAL_SAMPLES = len(X_val)\n    # Shapes\n    print(f'X_train shape: {X_train.shape}, X_val shape: {X_val.shape}')\n# Train On All Data\nelse:\n    # TRAIN\n    X_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/X.npy')\n    y_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/y.npy')[:,:MAX_PHRASE_LENGTH]\n    N_TRAIN_SAMPLES = len(X_train)\n    print(f'X_train shape: {X_train.shape}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Batch","metadata":{}},{"cell_type":"code","source":"# Example Batch For Debugging\nN_EXAMPLE_BATCH_SAMPLES = 1024\nN_EXAMPLE_BATCH_SAMPLES_SMALL = 32\n# Example Batch\nX_batch = {\n    'frames': np.copy(X_train[:N_EXAMPLE_BATCH_SAMPLES]),\n    'phrase': np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES]),\n#     'phrase_type': np.copy(y_phrase_type_train[:N_EXAMPLE_BATCH_SAMPLES]),\n}\ny_batch = np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES])\n# Small Example Batch\nX_batch_small = {\n    'frames': np.copy(X_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n    'phrase': np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n#     'phrase_type': np.copy(y_phrase_type_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n}\ny_batch_small = np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL])","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.116414Z","iopub.execute_input":"2024-04-01T12:35:26.116708Z","iopub.status.idle":"2024-04-01T12:35:26.155179Z","shell.execute_reply.started":"2024-04-01T12:35:26.116682Z","shell.execute_reply":"2024-04-01T12:35:26.154383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Parquet","metadata":{}},{"cell_type":"code","source":"# Read First Parquet File\n# example_parquet_df = pd.read_parquet(train['file_path'][0])\nexample_parquet_df = pd.read_parquet(INFERENCE_FILE_PATHS[0])\n\n# Each parquet file contains 1000 recordings\nprint(f'# Unique Recording: {example_parquet_df.index.nunique()}')\n# Display DataFrame layout\ndisplay(example_parquet_df.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.156279Z","iopub.execute_input":"2024-04-01T12:35:26.156543Z","iopub.status.idle":"2024-04-01T12:35:26.421691Z","shell.execute_reply.started":"2024-04-01T12:35:26.15652Z","shell.execute_reply":"2024-04-01T12:35:26.4207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Indices","metadata":{}},{"cell_type":"code","source":"# Get indices in original dataframe\ndef get_idxs(df, words_pos, words_neg=[], ret_names=True, idxs_pos=None):\n    idxs = []\n    names = []\n    for w in words_pos:\n        for col_idx, col in enumerate(example_parquet_df.columns):\n            # Exclude Non Landmark Columns\n            if col in ['frame']:\n                continue\n                \n            col_idx = int(col.split('_')[-1])\n            # Check if column name contains all words\n            if (w in col) and (idxs_pos is None or col_idx in idxs_pos) and all([w not in col for w in words_neg]):\n                idxs.append(col_idx)\n                names.append(col)\n    # Convert to Numpy arrays\n    idxs = np.array(idxs)\n    names = np.array(names)\n    # Returns either both column indices and names\n    if ret_names:\n        return idxs, names\n    # Or only columns indices\n    else:\n        return idxs","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.422843Z","iopub.execute_input":"2024-04-01T12:35:26.423096Z","iopub.status.idle":"2024-04-01T12:35:26.430808Z","shell.execute_reply.started":"2024-04-01T12:35:26.423074Z","shell.execute_reply":"2024-04-01T12:35:26.429854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lips Landmark Face Ids\nLIPS_LANDMARK_IDXS = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\n\n# Landmark Indices for Left/Right hand without z axis in raw data\nLEFT_HAND_IDXS0, LEFT_HAND_NAMES0 = get_idxs(example_parquet_df, ['left_hand'], ['z'])\nRIGHT_HAND_IDXS0, RIGHT_HAND_NAMES0 = get_idxs(example_parquet_df, ['right_hand'], ['z'])\nLIPS_IDXS0, LIPS_NAMES0 = get_idxs(example_parquet_df, ['face'], ['z'], idxs_pos=LIPS_LANDMARK_IDXS)\nCOLUMNS0 = np.concatenate((LEFT_HAND_NAMES0, RIGHT_HAND_NAMES0, LIPS_NAMES0))\nN_COLS0 = len(COLUMNS0)\n# Only X/Y axes are used\nN_DIMS0 = 2\n\nprint(f'N_COLS0: {N_COLS0}')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.432181Z","iopub.execute_input":"2024-04-01T12:35:26.432637Z","iopub.status.idle":"2024-04-01T12:35:26.443578Z","shell.execute_reply.started":"2024-04-01T12:35:26.432605Z","shell.execute_reply":"2024-04-01T12:35:26.442288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Landmark Indices in subset of dataframe with only COLUMNS selected\nLEFT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, LEFT_HAND_NAMES0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, RIGHT_HAND_NAMES0)).squeeze()\nLIPS_IDXS = np.argwhere(np.isin(COLUMNS0, LIPS_NAMES0)).squeeze()\nHAND_IDXS = np.concatenate((LEFT_HAND_IDXS, RIGHT_HAND_IDXS), axis=0)\nN_COLS = N_COLS0\n# Only X/Y axes are used\nN_DIMS = 2\n\nprint(f'N_COLS: {N_COLS}')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.444867Z","iopub.execute_input":"2024-04-01T12:35:26.445788Z","iopub.status.idle":"2024-04-01T12:35:26.454173Z","shell.execute_reply.started":"2024-04-01T12:35:26.445761Z","shell.execute_reply":"2024-04-01T12:35:26.453284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Indices in processed data by axes with only dominant hand\nHAND_X_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'x' in name]\n    ).squeeze()\nHAND_Y_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'y' in name]\n    ).squeeze()\n# Names in processed data by axes\nHAND_X_NAMES = LEFT_HAND_NAMES0[HAND_X_IDXS]\nHAND_Y_NAMES = LEFT_HAND_NAMES0[HAND_Y_IDXS]","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.455201Z","iopub.execute_input":"2024-04-01T12:35:26.455467Z","iopub.status.idle":"2024-04-01T12:35:26.463606Z","shell.execute_reply.started":"2024-04-01T12:35:26.455442Z","shell.execute_reply":"2024-04-01T12:35:26.462754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mean/STD Loading","metadata":{}},{"cell_type":"code","source":"# Mean/Standard Deviations of data used for normalizing\nMEANS = np.load('/kaggle/input/aslfr-preprocessing-dataset/MEANS.npy').reshape(-1)\nSTDS = np.load('/kaggle/input/aslfr-preprocessing-dataset/STDS.npy').reshape(-1)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.464761Z","iopub.execute_input":"2024-04-01T12:35:26.465113Z","iopub.status.idle":"2024-04-01T12:35:26.475054Z","shell.execute_reply.started":"2024-04-01T12:35:26.46508Z","shell.execute_reply":"2024-04-01T12:35:26.474262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorflow Preprocessing Layer","metadata":{}},{"cell_type":"code","source":"\"\"\"\n    Tensorflow layer to process data in TFLite\n    Data needs to be processed in the model itself, so we can not use Python\n\"\"\" \nclass PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n        self.normalisation_correction = tf.constant(\n                    # Add 0.50 to x coordinates of left hand (original right hand) and substract 0.50 of right hand (original left hand)\n                     [0.50 if 'x' in name else 0.00 for name in LEFT_HAND_NAMES0],\n                dtype=tf.float32,\n            )\n    \n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,N_COLS0], dtype=tf.float32),),\n    )\n    def call(self, data0, resize=True):\n        # Fill NaN Values With 0\n        data = tf.where(tf.math.is_nan(data0), 0.0, data0)\n        \n        # Hacky\n        data = data[None]\n        \n        # Empty Hand Frame Filtering\n        hands = tf.slice(data, [0,0,0], [-1, -1, 84])\n        hands = tf.abs(hands)\n        mask = tf.reduce_sum(hands, axis=2)\n        mask = tf.not_equal(mask, 0)\n        data = data[mask][None]\n        \n        # Pad Zeros\n        N_FRAMES = len(data[0])\n        if N_FRAMES < N_TARGET_FRAMES:\n            data = tf.concat((\n                data,\n                tf.zeros([1,N_TARGET_FRAMES-N_FRAMES,N_COLS], dtype=tf.float32)\n            ), axis=1)\n        # Downsample\n        data = tf.image.resize(\n            data,\n            [1, N_TARGET_FRAMES],\n            method=tf.image.ResizeMethod.BILINEAR,\n        )\n        \n        # Squeeze Batch Dimension\n        data = tf.squeeze(data, axis=[0])\n        \n        return data\n    \npreprocess_layer = PreprocessLayer()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.480422Z","iopub.execute_input":"2024-04-01T12:35:26.480679Z","iopub.status.idle":"2024-04-01T12:35:26.496019Z","shell.execute_reply.started":"2024-04-01T12:35:26.480657Z","shell.execute_reply":"2024-04-01T12:35:26.495193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function To Test Preprocessing Layer\ndef test_preprocess_layer():\n    demo_sequence_id = example_parquet_df.index.unique()[15]\n    demo_raw_data = example_parquet_df.loc[demo_sequence_id, COLUMNS0]\n    data = preprocess_layer(demo_raw_data)\n\n    print(f'demo_raw_data shape: {demo_raw_data.shape}')\n    print(f'data shape: {data.shape}')\n    \n    return data\n    \nif IS_INTERACTIVE:\n    data = test_preprocess_layer()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.497048Z","iopub.execute_input":"2024-04-01T12:35:26.497306Z","iopub.status.idle":"2024-04-01T12:35:26.648049Z","shell.execute_reply.started":"2024-04-01T12:35:26.497274Z","shell.execute_reply":"2024-04-01T12:35:26.647097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Dataset","metadata":{}},{"cell_type":"code","source":"# Train Dataset Iterator\ndef get_train_dataset(X, y, batch_size=BATCH_SIZE):\n    sample_idxs = np.arange(len(X))\n    while True:\n        # Get random indices\n        random_sample_idxs = np.random.choice(sample_idxs, batch_size)\n        \n        inputs = {\n            'frames': X[random_sample_idxs],\n            'phrase': y[random_sample_idxs],\n        }\n        outputs = y[random_sample_idxs]\n        \n        yield inputs, outputs","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.649224Z","iopub.execute_input":"2024-04-01T12:35:26.649521Z","iopub.status.idle":"2024-04-01T12:35:26.656531Z","shell.execute_reply.started":"2024-04-01T12:35:26.649495Z","shell.execute_reply":"2024-04-01T12:35:26.655403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Dataset\ntrain_dataset = get_train_dataset(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.65819Z","iopub.execute_input":"2024-04-01T12:35:26.658693Z","iopub.status.idle":"2024-04-01T12:35:26.663541Z","shell.execute_reply.started":"2024-04-01T12:35:26.658666Z","shell.execute_reply":"2024-04-01T12:35:26.66267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Steps Per Epoch\nTRAIN_STEPS_PER_EPOCH = math.ceil(N_TRAIN_SAMPLES / BATCH_SIZE)\nprint(f'TRAIN_STEPS_PER_EPOCH: {TRAIN_STEPS_PER_EPOCH}')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.664895Z","iopub.execute_input":"2024-04-01T12:35:26.665268Z","iopub.status.idle":"2024-04-01T12:35:26.673019Z","shell.execute_reply.started":"2024-04-01T12:35:26.665236Z","shell.execute_reply":"2024-04-01T12:35:26.672031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation Dataset","metadata":{}},{"cell_type":"code","source":"# Validation Set\ndef get_val_dataset(X, y, batch_size=BATCH_SIZE):\n    offsets = np.arange(0, len(X), batch_size)\n    while True:\n        # Iterate over whole validation set\n        for offset in offsets:\n            inputs = {\n                'frames': X[offset:offset+batch_size],\n                'phrase': y[offset:offset+batch_size],\n            }\n            outputs = y[offset:offset+batch_size]\n\n            yield inputs, outputs","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.674235Z","iopub.execute_input":"2024-04-01T12:35:26.675061Z","iopub.status.idle":"2024-04-01T12:35:26.681494Z","shell.execute_reply.started":"2024-04-01T12:35:26.675034Z","shell.execute_reply":"2024-04-01T12:35:26.680565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation Dataset\nif USE_VAL:\n    val_dataset = get_val_dataset(X_val, y_val)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.682751Z","iopub.execute_input":"2024-04-01T12:35:26.683338Z","iopub.status.idle":"2024-04-01T12:35:26.689834Z","shell.execute_reply.started":"2024-04-01T12:35:26.683306Z","shell.execute_reply":"2024-04-01T12:35:26.689057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    N_VAL_STEPS_PER_EPOCH = math.ceil(N_VAL_SAMPLES / BATCH_SIZE)\n    print(f'N_VAL_STEPS_PER_EPOCH: {N_VAL_STEPS_PER_EPOCH}')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.690906Z","iopub.execute_input":"2024-04-01T12:35:26.691204Z","iopub.status.idle":"2024-04-01T12:35:26.697625Z","shell.execute_reply.started":"2024-04-01T12:35:26.691178Z","shell.execute_reply":"2024-04-01T12:35:26.696809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Config","metadata":{}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# final embedding and transformer embedding size\nUNITS_ENCODER = 384\nUNITS_DECODER = 256\n\n# Transformer\nNUM_BLOCKS_ENCODER = 4\nNUM_BLOCKS_DECODER = 2\nNUM_HEADS = 4\nMLP_RATIO = 2\n\n# Dropout\nEMBEDDING_DROPOUT = 0.00\nMLP_DROPOUT_RATIO = 0.30\nMHA_DROPOUT_RATIO = 0.20\nCLASSIFIER_DROPOUT_RATIO = 0.10\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.698584Z","iopub.execute_input":"2024-04-01T12:35:26.698873Z","iopub.status.idle":"2024-04-01T12:35:26.706457Z","shell.execute_reply.started":"2024-04-01T12:35:26.69885Z","shell.execute_reply":"2024-04-01T12:35:26.705605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Embedding","metadata":{}},{"cell_type":"code","source":"# Embeds a landmark using fully connected layers\nclass LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n        self.supports_masking = True\n        \n    def build(self, input_shape):\n        # Embedding for missing landmark in frame, initizlied with zeros\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # Embedding\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.dense(x),\n            )","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.707718Z","iopub.execute_input":"2024-04-01T12:35:26.70829Z","iopub.status.idle":"2024-04-01T12:35:26.718639Z","shell.execute_reply.started":"2024-04-01T12:35:26.708258Z","shell.execute_reply":"2024-04-01T12:35:26.717822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Embedding","metadata":{}},{"cell_type":"code","source":"# Creates embedding for each frame\nclass Embedding(tf.keras.Model):\n    def __init__(self):\n        super(Embedding, self).__init__()\n        self.supports_masking = True\n    \n    def build(self, input_shape):\n        # Positional embedding for each frame index\n        self.positional_embedding = tf.Variable(\n            initial_value=tf.zeros([N_TARGET_FRAMES, UNITS_ENCODER], dtype=tf.float32),\n            trainable=True,\n            name='embedding_positional_encoder',\n        )\n        # Embedding layer for Landmarks\n        self.dominant_hand_embedding = LandmarkEmbedding(UNITS_ENCODER, 'dominant_hand')\n\n    def call(self, x, training=False):\n        # Normalize\n        x = tf.where(\n                tf.math.equal(x, 0.0),\n                0.0,\n                (x - MEANS) / STDS,\n            )\n        # Dominant Hand\n        x = self.dominant_hand_embedding(x)\n        # Add Positional Encoding\n        x = x + self.positional_embedding\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.719787Z","iopub.execute_input":"2024-04-01T12:35:26.720067Z","iopub.status.idle":"2024-04-01T12:35:26.727714Z","shell.execute_reply.started":"2024-04-01T12:35:26.720028Z","shell.execute_reply":"2024-04-01T12:35:26.726883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer","metadata":{}},{"cell_type":"code","source":"# based on: https://stackoverflow.com/questions/67342988/verifying-the-implementation-of-multihead-attention-in-transformer\n# replaced softmax with softmax layer to support masked softmax\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self, d_model, n_heads, dropout, d_out=None):\n        super(MultiHeadAttention,self).__init__()\n        # Number of Units in Model\n        self.d_model = d_model\n        # Number of Attention Heads\n        self.n_heads = n_heads\n        # Number of Units in Intermediate Layers\n        self.depth = d_model // 2\n        # Scaling Factor Of Values\n        self.scale = 1.0 / tf.math.sqrt(tf.cast(self.depth, tf.float32))\n        # Learnable Projections to Depth\n        self.wq = self.fused_mha(self.depth)\n        self.wk = self.fused_mha(self.depth)\n        self.wv = self.fused_mha(self.depth)\n        # Output Projection\n        self.wo = tf.keras.layers.Dense(d_model if d_out is None else d_out, use_bias=False)\n        # Softmax Activation Which Supports Masking\n        self.softmax = tf.keras.layers.Softmax()\n        # Reshaping Of Multiple Attention heads to Single Value\n        self.reshape = tf.keras.Sequential([\n            # [attention heads, number of frames, d_model] → [number of frames, n_heads, d_model // n_heads]\n            tf.keras.layers.Permute([2, 1, 3]),\n            # [number of frames, attention heads, d_model] → [number of frames, d_model]\n            tf.keras.layers.Reshape([N_TARGET_FRAMES, self.depth]),\n        ])\n        # Output Dropout\n        self.do = tf.keras.layers.Dropout(dropout)\n        self.supports_masking = True\n        \n    # Single dense layer for all attention heads\n    def fused_mha(self, dim):\n        return tf.keras.Sequential([\n            # Single dense layer\n            tf.keras.layers.Dense(dim, use_bias=False),\n            # Reshape to [number of frames, number of attention head, depth]\n            tf.keras.layers.Reshape([N_TARGET_FRAMES, self.n_heads, dim // self.n_heads]),\n            # Permutate to [number of attention heads, number of frames, depth]\n            tf.keras.layers.Permute([2, 1, 3]),\n        ])\n        \n    def call(self, q, k, v, attention_mask=None, training=False):\n        # Projections to attention heads\n        Q = self.wq(q)\n        K = self.wk(k)\n        V = self.wv(v)\n        # Matrix multiply QxK to acquire attention scores\n        x = tf.matmul(Q, K, transpose_b=True) * self.scale\n        # Softmax attention scores and Multiply with Values\n        x = self.softmax(x, mask=attention_mask) @ V\n        # Reshape to flatten attention heads\n        x = self.reshape(x)\n        # Output projection\n        x = self.wo(x)\n        # Dropout\n        x = self.do(x, training=training)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.729096Z","iopub.execute_input":"2024-04-01T12:35:26.729376Z","iopub.status.idle":"2024-04-01T12:35:26.74327Z","shell.execute_reply.started":"2024-04-01T12:35:26.729334Z","shell.execute_reply":"2024-04-01T12:35:26.742411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encoder\n\n[source](https://keras.io/examples/nlp/neural_machine_translation_with_transformer/)","metadata":{}},{"cell_type":"code","source":"# Encoder based on multiple transformer blocks\nclass Encoder(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Encoder, self).__init__(name='encoder')\n        self.num_blocks = num_blocks\n        self.supports_masking = True\n    \n    def build(self, input_shape):\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS_ENCODER, NUM_HEADS, MHA_DROPOUT_RATIO))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS_ENCODER * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS_ENCODER, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),\n            ]))\n            # Optional Projection to Decoder Dimension\n            if UNITS_ENCODER != UNITS_DECODER:\n                self.dense_out = tf.keras.layers.Dense(UNITS_DECODER, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False)\n                self.apply_dense_out = True\n            else:\n                self.apply_dense_out = False\n                \n    def get_attention_mask(self, x_inp):\n        # Attention Mask\n        attention_mask = tf.math.count_nonzero(x_inp, axis=[2], keepdims=True, dtype=tf.int32)\n        attention_mask = tf.math.count_nonzero(attention_mask, axis=[2], keepdims=False)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        return attention_mask\n        \n    def call(self, x, x_inp, training=False):\n        # Attention mask to ignore missing frames\n        attention_mask = self.get_attention_mask(x_inp)\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x = ln_1(x + mha(x, x, x, attention_mask=attention_mask))\n            x = ln_2(x + mlp(x))\n            \n        # Optional Projection to Decoder Dimension\n        if self.apply_dense_out:\n            x = self.dense_out(x)\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.744702Z","iopub.execute_input":"2024-04-01T12:35:26.744957Z","iopub.status.idle":"2024-04-01T12:35:26.758461Z","shell.execute_reply.started":"2024-04-01T12:35:26.744935Z","shell.execute_reply":"2024-04-01T12:35:26.75762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Decoder","metadata":{}},{"cell_type":"code","source":"# Decoder based on multiple transformer blocks\nclass Decoder(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Decoder, self).__init__(name='decoder')\n        self.num_blocks = num_blocks\n        self.supports_masking = True\n    \n    def build(self, input_shape):\n        # Causal Mask Batch Size 1\n        self.causal_mask = self.get_causal_attention_mask()\n        # Positional Embedding, initialized with zeros\n        self.positional_embedding = tf.Variable(\n            initial_value=tf.zeros([N_TARGET_FRAMES, UNITS_DECODER], dtype=tf.float32),\n            trainable=True,\n            name='embedding_positional_encoder',\n        )\n        # Character Embedding\n        self.char_emb = tf.keras.layers.Embedding(N_UNIQUE_CHARACTERS, UNITS_DECODER, embeddings_initializer=INIT_ZEROS)\n        # Positional Encoder MHA\n        self.pos_emb_mha = MultiHeadAttention(UNITS_DECODER, NUM_HEADS, MHA_DROPOUT_RATIO)\n        self.pos_emb_ln = tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS)\n        # First Layer Normalisation\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS_DECODER, NUM_HEADS, MHA_DROPOUT_RATIO))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS_DECODER * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS_DECODER, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),\n            ]))\n            \n    def get_causal_attention_mask(self):\n        i = tf.range(N_TARGET_FRAMES)[:, tf.newaxis]\n        j = tf.range(N_TARGET_FRAMES)\n        mask = tf.cast(i >= j, dtype=tf.int32)\n        mask = tf.reshape(mask, (1, N_TARGET_FRAMES, N_TARGET_FRAMES))\n        mult = tf.concat(\n            [tf.expand_dims(1, -1), tf.constant([1, 1], dtype=tf.int32)],\n            axis=0,\n        )\n        mask = tf.tile(mask, mult)\n        mask = tf.cast(mask, tf.float32)\n        return mask\n    \n    def get_attention_mask(self, x_inp):\n        # Attention Mask\n        attention_mask = tf.math.count_nonzero(x_inp, axis=[2], keepdims=True, dtype=tf.int32)\n        attention_mask = tf.math.count_nonzero(attention_mask, axis=[2], keepdims=False)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        return attention_mask\n        \n    def call(self, encoder_outputs, phrase, x_inp, training=False):\n        # Batch Size\n        B = tf.shape(encoder_outputs)[0]\n        # Cast to INT32\n        phrase = tf.cast(phrase, tf.int32)\n        # Prepend SOS Token\n        phrase = tf.pad(phrase, [[0,0], [1,0]], constant_values=SOS_TOKEN, name='prepend_sos_token')\n        # Pad With PAD Token\n        phrase = tf.pad(phrase, [[0,0], [0,N_TARGET_FRAMES-MAX_PHRASE_LENGTH-1]], constant_values=PAD_TOKEN, name='append_pad_token')\n        # Positional Embedding\n        x = self.positional_embedding + self.char_emb(phrase)\n        # Causal Attention\n        x = self.pos_emb_ln(x + self.pos_emb_mha(x, x, x, attention_mask=self.causal_mask))\n        # Attention mask to ignore missing frames\n        attention_mask = self.get_attention_mask(x_inp)\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x = ln_1(x + mha(x, encoder_outputs, encoder_outputs, attention_mask=attention_mask))\n            x = ln_2(x + mlp(x))\n        # Slice 31 Characters\n        x = tf.slice(x, [0, 0, 0], [-1, MAX_PHRASE_LENGTH, -1])\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.75973Z","iopub.execute_input":"2024-04-01T12:35:26.76Z","iopub.status.idle":"2024-04-01T12:35:26.781994Z","shell.execute_reply.started":"2024-04-01T12:35:26.759964Z","shell.execute_reply":"2024-04-01T12:35:26.781099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Causal Attention to make decoder not attent to future characters which it needs to predict\ndef get_causal_attention_mask(B):\n    i = tf.range(N_TARGET_FRAMES)[:, tf.newaxis]\n    j = tf.range(N_TARGET_FRAMES)\n    mask = tf.cast(i >= j, dtype=tf.int32)\n    mask = tf.reshape(mask, (1, N_TARGET_FRAMES, N_TARGET_FRAMES))\n    mult = tf.concat(\n        [tf.expand_dims(B, -1), tf.constant([1, 1], dtype=tf.int32)],\n        axis=0,\n    )\n    mask = tf.tile(mask, mult)\n    mask = tf.cast(mask, tf.float32)\n    return mask\n\nget_causal_attention_mask(1)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.783144Z","iopub.execute_input":"2024-04-01T12:35:26.78345Z","iopub.status.idle":"2024-04-01T12:35:26.800351Z","shell.execute_reply.started":"2024-04-01T12:35:26.783426Z","shell.execute_reply":"2024-04-01T12:35:26.799561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Non Pad/SOS/EOS Token Accuracy","metadata":{}},{"cell_type":"code","source":"# TopK accuracy for multi dimensional output\nclass TopKAccuracy(tf.keras.metrics.Metric):\n    def __init__(self, k, **kwargs):\n        super(TopKAccuracy, self).__init__(name=f'top{k}acc', **kwargs)\n        self.top_k_acc = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=k)\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        y_true = tf.reshape(y_true, [-1])\n        y_pred = tf.reshape(y_pred, [-1, N_UNIQUE_CHARACTERS])\n        character_idxs = tf.where(y_true < N_UNIQUE_CHARACTERS0)\n        y_true = tf.gather(y_true, character_idxs, axis=0)\n        y_pred = tf.gather(y_pred, character_idxs, axis=0)\n        self.top_k_acc.update_state(y_true, y_pred)\n\n    def result(self):\n        return self.top_k_acc.result()\n    \n    def reset_state(self):\n        self.top_k_acc.reset_state()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.80146Z","iopub.execute_input":"2024-04-01T12:35:26.801732Z","iopub.status.idle":"2024-04-01T12:35:26.809567Z","shell.execute_reply.started":"2024-04-01T12:35:26.801708Z","shell.execute_reply":"2024-04-01T12:35:26.808701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss Weights","metadata":{}},{"cell_type":"code","source":"# Create Initial Loss Weights All Set To 1\nloss_weights = np.ones(N_UNIQUE_CHARACTERS, dtype=np.float32)\n# Set Loss Weight Of Pad Token To 0\nloss_weights[PAD_TOKEN] = 0","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.810856Z","iopub.execute_input":"2024-04-01T12:35:26.811507Z","iopub.status.idle":"2024-04-01T12:35:26.821052Z","shell.execute_reply.started":"2024-04-01T12:35:26.811481Z","shell.execute_reply":"2024-04-01T12:35:26.820098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sparse Categorical Crossentropy With Label Smoothing¶","metadata":{}},{"cell_type":"code","source":"# source:: https://stackoverflow.com/questions/60689185/label-smoothing-for-sparse-categorical-crossentropy\ndef scce_with_ls(y_true, y_pred):\n    # Filter Pad Tokens\n    idxs = tf.where(y_true != PAD_TOKEN)\n    y_true = tf.gather_nd(y_true, idxs)\n    y_pred = tf.gather_nd(y_pred, idxs)\n    # One Hot Encode Sparsely Encoded Target Sign\n    y_true = tf.cast(y_true, tf.int32)\n    y_true = tf.one_hot(y_true, N_UNIQUE_CHARACTERS, axis=1)\n    # Categorical Crossentropy with native label smoothing support\n    loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, label_smoothing=0.25, from_logits=True)\n    loss = tf.math.reduce_mean(loss)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.822135Z","iopub.execute_input":"2024-04-01T12:35:26.822412Z","iopub.status.idle":"2024-04-01T12:35:26.830388Z","shell.execute_reply.started":"2024-04-01T12:35:26.822384Z","shell.execute_reply":"2024-04-01T12:35:26.829528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def get_model():\n    # Inputs\n    frames_inp = tf.keras.layers.Input([N_TARGET_FRAMES, N_COLS], dtype=tf.float32, name='frames')\n    phrase_inp = tf.keras.layers.Input([MAX_PHRASE_LENGTH], dtype=tf.int32, name='phrase')\n    # Frames\n    x = frames_inp\n\n    # Masking\n    x = tf.keras.layers.Masking(mask_value=0.0, input_shape=(N_TARGET_FRAMES, N_COLS))(x)\n    \n    # Embedding\n    x = Embedding()(x)\n    \n    # Encoder Transformer Blocks\n    x = Encoder(NUM_BLOCKS_ENCODER)(x, frames_inp)\n    \n    # Decoder\n    x = Decoder(NUM_BLOCKS_DECODER)(x, phrase_inp, frames_inp)\n    \n    # Classifier\n    x = tf.keras.Sequential([\n        # Dropout\n        tf.keras.layers.Dropout(CLASSIFIER_DROPOUT_RATIO),\n        # Output Neurons\n        tf.keras.layers.Dense(N_UNIQUE_CHARACTERS, activation=tf.keras.activations.linear, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),\n    ], name='classifier')(x)\n    \n    outputs = x\n    \n    # Create Tensorflow Model\n    model = tf.keras.models.Model(inputs=[frames_inp, phrase_inp], outputs=outputs)\n    \n    # Categorical Crossentropy Loss With Label Smoothing\n    loss = scce_with_ls\n    \n    # Adam Optimizer\n    optimizer = tfa.optimizers.RectifiedAdam(sma_threshold=4)\n    optimizer = tfa.optimizers.Lookahead(optimizer, sync_period=5)\n\n    # TopK Metrics\n    metrics = [\n        TopKAccuracy(1),\n        TopKAccuracy(5),\n    ]\n    \n    model.compile(\n        loss=loss,\n        optimizer=optimizer,\n        metrics=metrics,\n        loss_weights=loss_weights,\n    )\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.831711Z","iopub.execute_input":"2024-04-01T12:35:26.832263Z","iopub.status.idle":"2024-04-01T12:35:26.84231Z","shell.execute_reply.started":"2024-04-01T12:35:26.83223Z","shell.execute_reply":"2024-04-01T12:35:26.8414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Input data\nfor k, v in X_batch.items():\n    print(f'{k}: {v.shape}')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.843616Z","iopub.execute_input":"2024-04-01T12:35:26.844617Z","iopub.status.idle":"2024-04-01T12:35:26.854128Z","shell.execute_reply.started":"2024-04-01T12:35:26.844579Z","shell.execute_reply":"2024-04-01T12:35:26.853168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = get_model()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:26.855612Z","iopub.execute_input":"2024-04-01T12:35:26.856018Z","iopub.status.idle":"2024-04-01T12:35:29.381151Z","shell.execute_reply.started":"2024-04-01T12:35:26.855987Z","shell.execute_reply":"2024-04-01T12:35:29.380348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:29.382536Z","iopub.execute_input":"2024-04-01T12:35:29.382809Z","iopub.status.idle":"2024-04-01T12:35:29.53956Z","shell.execute_reply.started":"2024-04-01T12:35:29.382785Z","shell.execute_reply":"2024-04-01T12:35:29.538625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Model Architecture\ntf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True, show_layer_activations=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:29.540708Z","iopub.execute_input":"2024-04-01T12:35:29.540974Z","iopub.status.idle":"2024-04-01T12:35:29.668745Z","shell.execute_reply.started":"2024-04-01T12:35:29.540951Z","shell.execute_reply":"2024-04-01T12:35:29.667686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Verify Training Flag","metadata":{}},{"cell_type":"code","source":"def verify_correct_training_flag():\n    # Verify static output for inference\n    pred = model(X_batch_small, training=False)\n    for _ in tqdm(range(10)):\n        assert tf.reduce_min(tf.cast(pred == model(X_batch_small, training=False), tf.int8)) == 1\n\n    # Verify at least 99% varying output due to dropout during training\n    for _ in tqdm(range(10)):\n        assert tf.reduce_mean(tf.cast(pred != model(X_batch_small, training=True), tf.float32)) > 0.99\n        \nverify_correct_training_flag()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:29.670004Z","iopub.execute_input":"2024-04-01T12:35:29.670274Z","iopub.status.idle":"2024-04-01T12:35:32.89212Z","shell.execute_reply.started":"2024-04-01T12:35:29.67025Z","shell.execute_reply":"2024-04-01T12:35:32.891214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Verify No NaN Predictions","metadata":{}},{"cell_type":"code","source":"# Verify No NaN predictions\ndef verify_no_nan_predictions():\n    y_pred = model.predict(\n        val_dataset if USE_VAL else train_dataset,\n        steps=N_VAL_STEPS_PER_EPOCH if USE_VAL else 100,\n        verbose=VERBOSE,\n    )\n\n    print(f'# NaN Values In Predictions: {np.isnan(y_pred).sum()}')\n    \n    plt.figure(figsize=(15,8))\n    plt.title(f'Logit Predictions Initialized Model')\n    pd.Series(y_pred.flatten()).plot(kind='hist', bins=128)\n    plt.xlabel('Logits')\n    plt.grid()\n    plt.show()\n    \nverify_no_nan_predictions()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:32.901175Z","iopub.execute_input":"2024-04-01T12:35:32.901502Z","iopub.status.idle":"2024-04-01T12:35:40.281189Z","shell.execute_reply.started":"2024-04-01T12:35:32.901473Z","shell.execute_reply":"2024-04-01T12:35:40.280274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n    \n    if current_step < num_warmup_steps:\n        if WARMUP_METHOD == 'log':\n            return lr_max * 0.10 ** (num_warmup_steps - current_step)\n        else:\n            return lr_max * 2 ** -(num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:40.282698Z","iopub.execute_input":"2024-04-01T12:35:40.283067Z","iopub.status.idle":"2024-04-01T12:35:40.290022Z","shell.execute_reply.started":"2024-04-01T12:35:40.283033Z","shell.execute_reply":"2024-04-01T12:35:40.289155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule, epochs):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(1, epochs + 1)\n    x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n    plt.xlim([1, epochs])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n    # Increase y-limit for better readability\n    plt.ylim([0, max(lr_schedule) * 1.1])\n    \n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n            if x < len(lr_schedule) - 1:\n                if lr_schedule[x - 1] < val:\n                    ha = 'right'\n                else:\n                    ha = 'left'\n            elif x == 0:\n                ha = 'right'\n            else:\n                ha = 'left'\n            plt.plot(x + 1, val, 'o', color='black');\n            offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n            plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=0)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:40.291296Z","iopub.execute_input":"2024-04-01T12:35:40.291656Z","iopub.status.idle":"2024-04-01T12:35:40.659954Z","shell.execute_reply.started":"2024-04-01T12:35:40.291625Z","shell.execute_reply":"2024-04-01T12:35:40.659002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Decay Callback","metadata":{}},{"cell_type":"code","source":"# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:40.661181Z","iopub.execute_input":"2024-04-01T12:35:40.661501Z","iopub.status.idle":"2024-04-01T12:35:40.667336Z","shell.execute_reply.started":"2024-04-01T12:35:40.66147Z","shell.execute_reply":"2024-04-01T12:35:40.666437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate Initialized Model","metadata":{}},{"cell_type":"code","source":"# Evaluate Initialized Model On Validation Data\ny_pred = model.evaluate(\n    val_dataset if USE_VAL else train_dataset,\n    steps=N_VAL_STEPS_PER_EPOCH if USE_VAL else TRAIN_STEPS_PER_EPOCH,\n    verbose=VERBOSE,\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:35:40.668916Z","iopub.execute_input":"2024-04-01T12:35:40.669299Z","iopub.status.idle":"2024-04-01T12:36:25.413975Z","shell.execute_reply.started":"2024-04-01T12:35:40.669266Z","shell.execute_reply":"2024-04-01T12:36:25.412954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline","metadata":{}},{"cell_type":"code","source":"# baseline accuracy when only pad token is predicted\nif USE_VAL:\n    baseline_accuracy = np.mean(y_val == PAD_TOKEN)\nelse:\n    baseline_accuracy = np.mean(y_train == PAD_TOKEN)\nprint(f'Baseline Accuracy: {baseline_accuracy:.4f}')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:36:25.415508Z","iopub.execute_input":"2024-04-01T12:36:25.415893Z","iopub.status.idle":"2024-04-01T12:36:25.42656Z","shell.execute_reply.started":"2024-04-01T12:36:25.415859Z","shell.execute_reply":"2024-04-01T12:36:25.425591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:36:25.427641Z","iopub.execute_input":"2024-04-01T12:36:25.427901Z","iopub.status.idle":"2024-04-01T12:36:26.437193Z","shell.execute_reply.started":"2024-04-01T12:36:25.427879Z","shell.execute_reply":"2024-04-01T12:36:26.43624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n    # Clear all models in GPU\n    tf.keras.backend.clear_session()\n\n    # Get new fresh model\n    model = get_model()\n\n    # Sanity Check\n    model.summary()\n\n    # Actual Training\n    history = model.fit(\n            x=train_dataset,\n            steps_per_epoch=TRAIN_STEPS_PER_EPOCH,\n            epochs=N_EPOCHS,\n            # Only used for validation data since training data is a generator\n            validation_data=val_dataset if USE_VAL else None,\n            validation_steps=N_VAL_STEPS_PER_EPOCH if USE_VAL else None,\n            callbacks=[\n                lr_callback,\n                WeightDecayCallback(),\n            ],\n            verbose=VERBOSE,\n        )","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:36:26.444094Z","iopub.execute_input":"2024-04-01T12:36:26.444375Z","iopub.status.idle":"2024-04-01T12:40:44.057539Z","shell.execute_reply.started":"2024-04-01T12:36:26.444341Z","shell.execute_reply":"2024-04-01T12:40:44.056722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load Weights\nif LOAD_WEIGHTS:\n    model.load_weights('/kaggle/input/aslfr-training-python37/model.h5')\n    print(f'Successfully Loaded Pretrained Weights')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:40:44.058876Z","iopub.execute_input":"2024-04-01T12:40:44.059167Z","iopub.status.idle":"2024-04-01T12:40:44.065847Z","shell.execute_reply.started":"2024-04-01T12:40:44.059141Z","shell.execute_reply":"2024-04-01T12:40:44.065004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save Model Weights\nmodel.save_weights('model.h5')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:40:44.067037Z","iopub.execute_input":"2024-04-01T12:40:44.067667Z","iopub.status.idle":"2024-04-01T12:40:44.186007Z","shell.execute_reply.started":"2024-04-01T12:40:44.067634Z","shell.execute_reply":"2024-04-01T12:40:44.18506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify Model is Loaded Correctly\nmodel.evaluate(\n    val_dataset if USE_VAL else train_dataset,\n    steps=N_VAL_STEPS_PER_EPOCH if USE_VAL else TRAIN_STEPS_PER_EPOCH,\n    batch_size=BATCH_SIZE,\n    verbose=VERBOSE,\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:40:44.187118Z","iopub.execute_input":"2024-04-01T12:40:44.187417Z","iopub.status.idle":"2024-04-01T12:41:28.833205Z","shell.execute_reply.started":"2024-04-01T12:40:44.187381Z","shell.execute_reply":"2024-04-01T12:41:28.832313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Levenshtein Distance","metadata":{}},{"cell_type":"code","source":"# Output Predictions to string\ndef outputs2phrase(outputs):\n    if outputs.ndim == 2:\n        outputs = np.argmax(outputs, axis=1)\n    \n    return ''.join([ORD2CHAR.get(s, '') for s in outputs])","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:28.834342Z","iopub.execute_input":"2024-04-01T12:41:28.834653Z","iopub.status.idle":"2024-04-01T12:41:28.841515Z","shell.execute_reply.started":"2024-04-01T12:41:28.834627Z","shell.execute_reply":"2024-04-01T12:41:28.840444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function(jit_compile=True)\ndef predict_phrase(frames):\n    # Add Batch Dimension\n    frames = tf.expand_dims(frames, axis=0)\n    # Start Phrase\n    phrase = tf.fill([1,MAX_PHRASE_LENGTH], PAD_TOKEN)\n\n    for idx in tf.range(MAX_PHRASE_LENGTH):\n        # Cast phrase to int8\n        phrase = tf.cast(phrase, tf.int8)\n        # Predict Next Token\n        outputs = model({\n            'frames': frames,\n            'phrase': phrase,\n        })\n\n        # Add predicted token to input phrase\n        phrase = tf.cast(phrase, tf.int32)\n        phrase = tf.where(\n            tf.range(MAX_PHRASE_LENGTH) < idx + 1,\n            tf.argmax(outputs, axis=2, output_type=tf.int32),\n            phrase,\n        )\n\n    # Squeeze outputs\n    outputs = tf.squeeze(phrase, axis=0)\n    outputs = tf.one_hot(outputs, N_UNIQUE_CHARACTERS)\n\n    # Return a dictionary with the output tensor\n    return outputs\n\n    # Return a dictionary with the output tensor\n    return outputs","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:28.842634Z","iopub.execute_input":"2024-04-01T12:41:28.842897Z","iopub.status.idle":"2024-04-01T12:41:28.858993Z","shell.execute_reply.started":"2024-04-01T12:41:28.842875Z","shell.execute_reply":"2024-04-01T12:41:28.858061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Levenstein Distance Train","metadata":{}},{"cell_type":"code","source":"# Compute Levenstein Distances\ndef get_ld_train():\n    N = 100 if IS_INTERACTIVE else 1000\n    LD_TRAIN = []\n    for idx, (frames, phrase_true) in enumerate(zip(tqdm(X_train, total=N), y_train)):\n        # Predict Phrase and Convert to String\n        phrase_pred = predict_phrase(frames).numpy()\n        phrase_pred = outputs2phrase(phrase_pred)\n        # True Phrase Ordinal to String\n        phrase_true = outputs2phrase(phrase_true)\n        # Add Levenstein Distance\n        LD_TRAIN.append({\n            'phrase_true': phrase_true,\n            'phrase_true_len': len(phrase_true),\n            'phrase_pred': phrase_pred,\n            'levenshtein_distance': levenshtein(phrase_pred, phrase_true),\n        })\n        # Take subset in interactive mode\n        if idx == N:\n            break\n            \n    # Convert to DataFrame\n    LD_TRAIN_DF = pd.DataFrame(LD_TRAIN)\n    \n    return LD_TRAIN_DF","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:28.860587Z","iopub.execute_input":"2024-04-01T12:41:28.860841Z","iopub.status.idle":"2024-04-01T12:41:28.867804Z","shell.execute_reply.started":"2024-04-01T12:41:28.860818Z","shell.execute_reply":"2024-04-01T12:41:28.86681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LD_TRAIN_DF = get_ld_train()\n\n# Display Errors\ndisplay(LD_TRAIN_DF.head(30))","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:28.869042Z","iopub.execute_input":"2024-04-01T12:41:28.869418Z","iopub.status.idle":"2024-04-01T12:41:34.985942Z","shell.execute_reply.started":"2024-04-01T12:41:28.869385Z","shell.execute_reply":"2024-04-01T12:41:34.984935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Value Counts\nLD_TRAIN_VC = dict([(i, 0) for i in range(LD_TRAIN_DF['levenshtein_distance'].max()+1)])\nfor ld in LD_TRAIN_DF['levenshtein_distance']:\n    LD_TRAIN_VC[ld] += 1\n\n# Evaluation Metric\nN = LD_TRAIN_DF['phrase_true_len'].sum()\nD = LD_TRAIN_DF['levenshtein_distance'].sum()\nnld = (N - D) / N\n\nLD_TRAIN_VC = dict([(i, 0) for i in range(LD_TRAIN_DF['levenshtein_distance'].max()+1)])\nfor ld in LD_TRAIN_DF['levenshtein_distance']:\n    LD_TRAIN_VC[ld] += 1\n\nplt.figure(figsize=(15,8))\npd.Series(LD_TRAIN_VC).plot(kind='bar', width=1)\nplt.title(f'Train Levenstein Distance Distribution | Mean: {LD_TRAIN_DF.levenshtein_distance.mean():.4f}, NLD: {nld:.3f}')\nplt.xlabel('Levenstein Distance')\nplt.ylabel('Sample Count')\nplt.xlim(-0.50, LD_TRAIN_DF.levenshtein_distance.max()+0.50)\nplt.grid(axis='y')\nplt.savefig('temp.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:34.98724Z","iopub.execute_input":"2024-04-01T12:41:34.987561Z","iopub.status.idle":"2024-04-01T12:41:35.72432Z","shell.execute_reply.started":"2024-04-01T12:41:34.987534Z","shell.execute_reply":"2024-04-01T12:41:35.723379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Levenstein Distance Evaluation","metadata":{}},{"cell_type":"code","source":"# Compute Levenstein Distances\ndef get_ld_val():\n    N = 100 if IS_INTERACTIVE else 1000\n    LD_VAL = []\n    for idx, (frames, phrase_true) in enumerate(zip(tqdm(X_val, total=N), y_val)):\n        # Predict Phrase and Convert to String\n        phrase_pred = predict_phrase(frames).numpy()\n        phrase_pred = outputs2phrase(phrase_pred)\n        # True Phrase Ordinal to String\n        phrase_true = outputs2phrase(phrase_true)\n        # Add Levenstein Distance\n        LD_VAL.append({\n            'phrase_true': phrase_true,\n            'phrase_true_len': len(phrase_true),\n            'phrase_pred': phrase_pred,\n            'levenshtein_distance': levenshtein(phrase_pred, phrase_true),\n        })\n        # Take subset in interactive mode\n        if idx == N:\n            break\n            \n    # Convert to DataFrame\n    LD_VAL_DF = pd.DataFrame(LD_VAL)\n    \n    return LD_VAL_DF","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:35.725718Z","iopub.execute_input":"2024-04-01T12:41:35.726087Z","iopub.status.idle":"2024-04-01T12:41:35.73364Z","shell.execute_reply.started":"2024-04-01T12:41:35.726053Z","shell.execute_reply":"2024-04-01T12:41:35.73275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    LD_VAL_DF = get_ld_val()\n\n    # Display Errors\n    display(LD_VAL_DF.head(30))","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:35.734925Z","iopub.execute_input":"2024-04-01T12:41:35.735226Z","iopub.status.idle":"2024-04-01T12:41:35.745514Z","shell.execute_reply.started":"2024-04-01T12:41:35.735191Z","shell.execute_reply":"2024-04-01T12:41:35.744623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Value Counts\nif USE_VAL:\n    # Evaluation Metric\n    N = LD_VAL_DF['phrase_true_len'].sum()\n    D = LD_VAL_DF['levenshtein_distance'].sum()\n    nld = (N - D) / N\n    \n    LD_VAL_VC = dict([(i, 0) for i in range(LD_VAL_DF['levenshtein_distance'].max()+1)])\n    for ld in LD_VAL_DF['levenshtein_distance']:\n        LD_VAL_VC[ld] += 1\n\n    plt.figure(figsize=(15,8))\n    pd.Series(LD_VAL_VC).plot(kind='bar', width=1)\n    plt.title(f'Validation Levenstein Distance Distribution | Mean: {LD_VAL_DF.levenshtein_distance.mean():.4f}, NLD: {nld:.3f}')\n    plt.xlabel('Levenstein Distance')\n    plt.ylabel('Sample Count')\n    plt.xlim(-0.50, LD_VAL_DF.levenshtein_distance.max()+0.50)\n    plt.grid(axis='y')\n    plt.savefig('temp.png')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:35.746812Z","iopub.execute_input":"2024-04-01T12:41:35.747719Z","iopub.status.idle":"2024-04-01T12:41:35.755117Z","shell.execute_reply.started":"2024-04-01T12:41:35.747684Z","shell.execute_reply":"2024-04-01T12:41:35.754398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax, ylim=None, yscale=None, yticks=None):\n    # Only plot when training\n    if not TRAIN_MODEL:\n        return\n    \n    plt.figure(figsize=(20, 10))\n    \n    values = history.history[metric]\n    N_EPOCHS = len(values)\n    val = 'val' in ''.join(history.history.keys())\n    # Epoch Ticks\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n\n    x_ticks = np.arange(1, N_EPOCHS+1)\n\n    # Validation\n    if val:\n        val_values = history.history[f'val_{metric}']\n        val_argmin = f_best(val_values)\n        plt.plot(x_ticks, val_values, label=f'val')\n\n    # summarize history for accuracy\n    plt.plot(x_ticks, values, label=f'train')\n    argmin = f_best(values)\n    plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label=f'train_best')\n    if val:\n        plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label=f'val_best')\n\n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n\n    if ylim:\n        plt.ylim(ylim)\n\n    if yscale is not None:\n        plt.yscale(yscale)\n        \n    if yticks is not None:\n        plt.yticks(yticks, fontsize=16)\n\n    plt.xlabel('epoch', fontsize=20, labelpad=10)        \n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    \n    plt.legend(prop={'size': 10})\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:35.756243Z","iopub.execute_input":"2024-04-01T12:41:35.756515Z","iopub.status.idle":"2024-04-01T12:41:35.769292Z","shell.execute_reply.started":"2024-04-01T12:41:35.756492Z","shell.execute_reply":"2024-04-01T12:41:35.768527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:35.770383Z","iopub.execute_input":"2024-04-01T12:41:35.770698Z","iopub.status.idle":"2024-04-01T12:41:36.170836Z","shell.execute_reply.started":"2024-04-01T12:41:35.770673Z","shell.execute_reply":"2024-04-01T12:41:36.169903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('top1acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:36.172058Z","iopub.execute_input":"2024-04-01T12:41:36.172341Z","iopub.status.idle":"2024-04-01T12:41:36.555745Z","shell.execute_reply.started":"2024-04-01T12:41:36.172316Z","shell.execute_reply":"2024-04-01T12:41:36.554798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('top5acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:36.557124Z","iopub.execute_input":"2024-04-01T12:41:36.55779Z","iopub.status.idle":"2024-04-01T12:41:36.937108Z","shell.execute_reply.started":"2024-04-01T12:41:36.557755Z","shell.execute_reply":"2024-04-01T12:41:36.936151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"# Model Layer Names\nfor l in model.layers:\n    print(l.name)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:36.938257Z","iopub.execute_input":"2024-04-01T12:41:36.938563Z","iopub.status.idle":"2024-04-01T12:41:36.943791Z","shell.execute_reply.started":"2024-04-01T12:41:36.938537Z","shell.execute_reply":"2024-04-01T12:41:36.942777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TFLite model for submission\nclass TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n\n        # Load the feature generation and main models\n        self.preprocess_layer = preprocess_layer\n        self.model = model\n    \n    @tf.function(jit_compile=True)\n    def encoder(self, x, frames_inp):\n        x = self.model.get_layer('embedding')(x)\n        x = self.model.get_layer('encoder')(x, frames_inp)\n        \n        return x\n        \n    @tf.function(jit_compile=True)\n    def decoder(self, x, phrase_inp, frames_inp):\n        x = self.model.get_layer('decoder')(x, phrase_inp, frames_inp)\n        x = self.model.get_layer('classifier')(x)\n        \n        return x\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, N_COLS0], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs):\n        # Number Of Input Frames\n        N_INPUT_FRAMES = tf.shape(inputs)[0]\n        # Preprocess Data\n        frames_inp = self.preprocess_layer(inputs)        \n        # Add Batch Dimension\n        frames_inp = tf.expand_dims(frames_inp, axis=0)\n        # Get Encoding\n        encoding = self.encoder(frames_inp, frames_inp)\n        # Make Prediction\n        phrase = tf.fill([1,MAX_PHRASE_LENGTH], PAD_TOKEN)\n        # Predict One Token At A Time\n        stop = False\n        for idx in tf.range(MAX_PHRASE_LENGTH):\n            # Cast phrase to int8\n            phrase = tf.cast(phrase, tf.int8)\n            # If EOS token is predicted, stop predicting\n            outputs = tf.cond(\n                stop,\n                lambda: tf.one_hot(tf.cast(phrase, tf.int32), N_UNIQUE_CHARACTERS),\n                lambda: self.decoder(encoding, phrase, frames_inp)\n            )\n            # Add predicted token to input phrase\n            phrase = tf.cast(phrase, tf.int32)\n            # Replcae PAD token with predicted token up to idx\n            phrase = tf.where(\n                tf.range(MAX_PHRASE_LENGTH) < idx + 1,\n                tf.argmax(outputs, axis=2, output_type=tf.int32),\n                phrase,\n            )\n            # Predicted Token\n            predicted_token = phrase[0,idx]\n            # If EOS (End Of Sentence) token is predicted stop\n            if not stop:\n                stop = predicted_token == EOS_TOKEN\n            \n        # Squeeze outputs\n        outputs = tf.squeeze(phrase, axis=0)\n        outputs = tf.one_hot(outputs, N_UNIQUE_CHARACTERS)\n            \n        # Return a dictionary with the output tensor\n        return {'outputs': outputs }\n\n# Define TF Lite Model\ntflite_keras_model = TFLiteModel(model)\n\n# Sanity Check\n# demo_sequence_id = 1816796431\ndemo_sequence_id = example_parquet_df.index.unique()[0]\ndemo_raw_data = example_parquet_df.loc[demo_sequence_id, COLUMNS0].values\ndemo_phrase_true = train_sequence_id.loc[demo_sequence_id, 'phrase']\nprint(f'demo_raw_data shape: {demo_raw_data.shape}, dtype: {demo_raw_data.dtype}')\ndemo_output = tflite_keras_model(demo_raw_data)['outputs'].numpy()\nprint(f'demo_output shape: {demo_output.shape}, dtype: {demo_output.dtype}')\nprint(f'demo_outputs phrase decoded: {outputs2phrase(demo_output)}')\nprint(f'phrase true: {demo_phrase_true}')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:36.945214Z","iopub.execute_input":"2024-04-01T12:41:36.94579Z","iopub.status.idle":"2024-04-01T12:41:39.90137Z","shell.execute_reply.started":"2024-04-01T12:41:36.945766Z","shell.execute_reply":"2024-04-01T12:41:39.900418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Model Converter\nkeras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\n# Convert Model\ntflite_model = keras_model_converter.convert()\n# Write Model\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:41:39.902576Z","iopub.execute_input":"2024-04-01T12:41:39.902856Z","iopub.status.idle":"2024-04-01T12:42:42.417016Z","shell.execute_reply.started":"2024-04-01T12:41:39.902831Z","shell.execute_reply":"2024-04-01T12:42:42.415946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add selected_columns json to only select specific columns from input frames\nwith open('inference_args.json', 'w') as f:\n     json.dump({ 'selected_columns': COLUMNS0.tolist() }, f)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:42:42.418476Z","iopub.execute_input":"2024-04-01T12:42:42.418773Z","iopub.status.idle":"2024-04-01T12:42:42.42421Z","shell.execute_reply.started":"2024-04-01T12:42:42.418748Z","shell.execute_reply":"2024-04-01T12:42:42.423259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}