{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U --no-deps leven cached-property","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:19:40.113312Z","iopub.execute_input":"2023-08-18T20:19:40.113691Z","iopub.status.idle":"2023-08-18T20:19:42.764124Z","shell.execute_reply.started":"2023-08-18T20:19:40.113661Z","shell.execute_reply":"2023-08-18T20:19:42.762933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from shutil import copyfile\n\n# copy our file into the working directory (make sure it has .py suffix)\ncopyfile(src = \"/kaggle/input/ctc-tpu/CTC_TPU.py\", dst = \"/kaggle/working//CTC_TPU.py\")\n\n# import all our functions\nfrom CTC_TPU import classic_ctc_loss","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:19:42.766848Z","iopub.execute_input":"2023-08-18T20:19:42.767242Z","iopub.status.idle":"2023-08-18T20:19:54.814984Z","shell.execute_reply.started":"2023-08-18T20:19:42.767205Z","shell.execute_reply":"2023-08-18T20:19:54.813944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport tensorflow as tf\nimport tensorflow_addons as tfa\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit\nfrom leven import levenshtein\n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport time\nimport json\n\n# Maximum Columns Pandas\npd.options.display.max_rows = 999\n\nprint(f'Tensorflow Version {tf.__version__}')\nprint(f'Python Version: {sys.version}')","metadata":{"papermill":{"duration":42.843261,"end_time":"2023-07-03T18:20:07.38955","exception":false,"start_time":"2023-07-03T18:19:24.546289","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:19:54.816515Z","iopub.execute_input":"2023-08-18T20:19:54.817613Z","iopub.status.idle":"2023-08-18T20:19:55.834428Z","shell.execute_reply.started":"2023-08-18T20:19:54.817578Z","shell.execute_reply":"2023-08-18T20:19:55.833488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hardware Config","metadata":{"papermill":{"duration":0.027883,"end_time":"2023-07-03T18:20:07.445719","exception":false,"start_time":"2023-07-03T18:20:07.417836","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=\"local\") # \"local\" for 1VM TPU\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept tf.errors.NotFoundError:\n    strategy = tf.distribute.MirroredStrategy()\n    \nN_REPLICAS = strategy.num_replicas_in_sync    \nprint(f'REPLICAS: {N_REPLICAS}')","metadata":{"papermill":{"duration":9.020791,"end_time":"2023-07-03T18:20:16.494441","exception":false,"start_time":"2023-07-03T18:20:07.47365","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:19:55.839844Z","iopub.execute_input":"2023-08-18T20:19:55.842474Z","iopub.status.idle":"2023-08-18T20:19:56.150273Z","shell.execute_reply.started":"2023-08-18T20:19:55.842438Z","shell.execute_reply":"2023-08-18T20:19:56.149384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Character 2 Ordinal Encoding","metadata":{"papermill":{"duration":0.029315,"end_time":"2023-07-03T18:20:16.553686","exception":false,"start_time":"2023-07-03T18:20:16.524371","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Read Character to Ordinal Encoding Mapping\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    CHAR2ORD = json.load(json_file)\n    \n# Ordinal to Character Mapping\nORD2CHAR = {j:i for i,j in CHAR2ORD.items()}\n    \n# Character to Ordinal Encoding Mapping   \ndisplay(pd.Series(CHAR2ORD).to_frame('Ordinal Encoding'))","metadata":{"papermill":{"duration":0.05459,"end_time":"2023-07-03T18:20:16.637506","exception":false,"start_time":"2023-07-03T18:20:16.582916","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:19:56.154862Z","iopub.execute_input":"2023-08-18T20:19:56.1572Z","iopub.status.idle":"2023-08-18T20:19:56.191382Z","shell.execute_reply.started":"2023-08-18T20:19:56.157164Z","shell.execute_reply":"2023-08-18T20:19:56.190518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Global Config","metadata":{"papermill":{"duration":0.029353,"end_time":"2023-07-03T18:20:16.69748","exception":false,"start_time":"2023-07-03T18:20:16.668127","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# If Notebook Is Run By Committing or In Interactive Mode For Development\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n# Verbose Setting during training\nVERBOSE = 1 if IS_INTERACTIVE else 2\n# Global Random Seed\nSEED = 42\n# Number of Frames to resize recording to\nN_TARGET_FRAMES = 288\n# Global debug flag, takes subset of train\nDEBUG = False\n# Number of Unique Characters To Predict + Pad Token + SOS Token + EOS Token\nN_UNIQUE_CHARACTERS0 = len(CHAR2ORD)\nN_UNIQUE_CHARACTERS = len(CHAR2ORD) + 1 + 1 + 1\nPAD_TOKEN = len(CHAR2ORD) # Padding\nSOS_TOKEN = len(CHAR2ORD) + 1 # Start Of Sentence\nEOS_TOKEN = len(CHAR2ORD) + 2 # End Of Sentence\n# Whether to use 10% of data for validation\nUSE_VAL = True\n# Batch Size\nBATCH_SIZE = 32\n# Weight Decay Ratio as Ratio of Learning Rate\nWD_RATIO = 0.05\n# Maximum Learning Rate\nLR_MAX = 4e-3\n# Warmup Method\nWARMUP_METHOD = 'exp'\n# Whether to Train The model\nTRAIN_MODEL = True\n# Whether to Load Pretrained Weights\nLOAD_WEIGHTS = False\n# Dataset Scalar float32 -> int16\nDATASET_SCALAR = 1e4\n# Competition or Supplementary Dataset\nDATASET = 'competition'\n# Number of Phrase Types\nN_PHRASE_TYPES = 3\n# Disable TQDM on TPU\nDISABLE_TQDM = False\n# Tensorflow Datasets AUTO Flag\nAUTO = tf.data.AUTOTUNE\n# Global Epsilon Value\nEPS = tf.keras.backend.epsilon()","metadata":{"papermill":{"duration":0.041228,"end_time":"2023-07-03T18:20:16.768273","exception":false,"start_time":"2023-07-03T18:20:16.727045","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:19:56.195585Z","iopub.execute_input":"2023-08-18T20:19:56.197891Z","iopub.status.idle":"2023-08-18T20:19:56.209149Z","shell.execute_reply.started":"2023-08-18T20:19:56.197857Z","shell.execute_reply":"2023-08-18T20:19:56.208169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"papermill":{"duration":0.029357,"end_time":"2023-07-03T18:20:16.827455","exception":false,"start_time":"2023-07-03T18:20:16.798098","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if DATASET == 'competition':\n    # Dataset Folder\n    DATASET_FOLDER = '/kaggle/input/fsfr-dataset-generation-python37'\n    # Number of Epochs to Train for\n    N_EPOCHS = 2 if IS_INTERACTIVE else 200\n    # Length of Phrase + EOS Token\n    MAX_PHRASE_LENGTH = 288\n    # Maximum Prediction Length\n    MAX_PRED_LENGTH = 32\n    # Number of Warmup Epochs in Learning Rate Scheduler\n    N_WARMUP_EPOCHS = 10\nelif DATASET == 'supplemental':\n    # Dataset Folder\n    DATASET_FOLDER = '/kaggle/input/fsfr-dataset-generation-supplemental-python37'\n    # Number of Epochs to Train for\n    N_EPOCHS = 2 if IS_INTERACTIVE else 50\n    # Length of Phrase + EOS Token\n    MAX_PHRASE_LENGTH = 43 + 1\n    # Number of Warmup Epochs in Learning Rate Scheduler\n    N_WARMUP_EPOCHS = 0","metadata":{"papermill":{"duration":0.038161,"end_time":"2023-07-03T18:20:16.895157","exception":false,"start_time":"2023-07-03T18:20:16.856996","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:19:56.214596Z","iopub.execute_input":"2023-08-18T20:19:56.218017Z","iopub.status.idle":"2023-08-18T20:19:56.226129Z","shell.execute_reply.started":"2023-08-18T20:19:56.217961Z","shell.execute_reply":"2023-08-18T20:19:56.225163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Config","metadata":{"papermill":{"duration":0.02946,"end_time":"2023-07-03T18:20:16.954241","exception":false,"start_time":"2023-07-03T18:20:16.924781","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"papermill":{"duration":0.038838,"end_time":"2023-07-03T18:20:17.022701","exception":false,"start_time":"2023-07-03T18:20:16.983863","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:19:56.231413Z","iopub.execute_input":"2023-08-18T20:19:56.234336Z","iopub.status.idle":"2023-08-18T20:19:56.243398Z","shell.execute_reply.started":"2023-08-18T20:19:56.234292Z","shell.execute_reply":"2023-08-18T20:19:56.242244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{"papermill":{"duration":0.029668,"end_time":"2023-07-03T18:20:17.081959","exception":false,"start_time":"2023-07-03T18:20:17.052291","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Read Train DataFrame\nif DEBUG:\n    if DATASET == 'competition':\n        train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv').head(5000)\n    elif DATASET == 'supplemental':\n        train = pd.read_csv('/kaggle/input/asl-fingerspelling/supplemental_metadata.csv').head(5000)\nelse:\n    if DATASET == 'competition':\n        train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n    elif DATASET == 'supplemental':\n        train = pd.read_csv('/kaggle/input/asl-fingerspelling/supplemental_metadata.csv')\n    \n# Set Train Indexed By sqeuence_id\ntrain_sequence_id = train.set_index('sequence_id')\n\n# Number Of Train Samples\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}')\n\ndisplay(train.info())\ndisplay(train.head())","metadata":{"papermill":{"duration":0.192971,"end_time":"2023-07-03T18:20:17.304976","exception":false,"start_time":"2023-07-03T18:20:17.112005","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:19:56.24833Z","iopub.execute_input":"2023-08-18T20:19:56.250972Z","iopub.status.idle":"2023-08-18T20:19:56.523408Z","shell.execute_reply.started":"2023-08-18T20:19:56.25094Z","shell.execute_reply":"2023-08-18T20:19:56.522522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# File Path","metadata":{"papermill":{"duration":0.032326,"end_time":"2023-07-03T18:20:17.372697","exception":false,"start_time":"2023-07-03T18:20:17.340371","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-fingerspelling/{path}'\n\ntrain['file_path'] = train['path'].apply(get_file_path)","metadata":{"papermill":{"duration":0.06017,"end_time":"2023-07-03T18:20:17.465633","exception":false,"start_time":"2023-07-03T18:20:17.405463","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:19:56.530592Z","iopub.execute_input":"2023-08-18T20:19:56.532822Z","iopub.status.idle":"2023-08-18T20:19:56.573629Z","shell.execute_reply.started":"2023-08-18T20:19:56.532789Z","shell.execute_reply":"2023-08-18T20:19:56.572702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example File Paths","metadata":{"papermill":{"duration":0.030117,"end_time":"2023-07-03T18:20:17.52822","exception":false,"start_time":"2023-07-03T18:20:17.498103","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Unique Parquet Files\nINFERENCE_FILE_PATHS = pd.Series(\n        glob.glob(f'{DATASET_FOLDER}/train_landmark_subsets/*')\n    )\n\nprint(f'Found {len(INFERENCE_FILE_PATHS)} Inference Pickle Files')","metadata":{"papermill":{"duration":0.0516,"end_time":"2023-07-03T18:20:17.60989","exception":false,"start_time":"2023-07-03T18:20:17.55829","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:19:56.578042Z","iopub.execute_input":"2023-08-18T20:19:56.580266Z","iopub.status.idle":"2023-08-18T20:19:56.598859Z","shell.execute_reply.started":"2023-08-18T20:19:56.580233Z","shell.execute_reply":"2023-08-18T20:19:56.598054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load X/y","metadata":{"papermill":{"duration":0.037453,"end_time":"2023-07-03T18:20:17.681289","exception":false,"start_time":"2023-07-03T18:20:17.643836","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Train/Validation\nif USE_VAL:\n    # VAL\n    X_val = np.load(f'{DATASET_FOLDER}/X_val.npy')\n    y_val = np.load(f'{DATASET_FOLDER}/y_val.npy')\n    n_frames_val = np.load(f'{DATASET_FOLDER}/n_frames_val.npy')\n    N_VAL_SAMPLES = len(X_val)\n    # Shapes\n    print(f'X_val shape: {X_val.shape}')\n# Train On All Data\nelse:\n    # TRAIN\n    X_train = np.load(f'{DATASET_FOLDER}/X_subset.npy')\n    y_train = np.load(f'{DATASET_FOLDER}/y_subset.npy')\n    n_frames_train = np.load(f'{DATASET_FOLDER}/n_frames_subset.npy')\n    N_TRAIN_SAMPLES = len(X_train)\n    print(f'X_train shape: {X_train.shape}')","metadata":{"papermill":{"duration":102.903497,"end_time":"2023-07-03T18:22:00.624359","exception":false,"start_time":"2023-07-03T18:20:17.720862","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:19:56.60295Z","iopub.execute_input":"2023-08-18T20:19:56.605179Z","iopub.status.idle":"2023-08-18T20:20:05.579653Z","shell.execute_reply.started":"2023-08-18T20:19:56.605146Z","shell.execute_reply":"2023-08-18T20:20:05.578666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    # Prepend SOS Token\n    y_val = np.load(f'{DATASET_FOLDER}/y_val.npy')\n    y_val_inp = np.pad(y_val, [[0,0],[1,0]], constant_values=SOS_TOKEN)\n    y_val_inp = y_val_inp[:,:N_TARGET_FRAMES]\nelse:\n    # Prepend SOS Token\n    y_train = np.load(f'{DATASET_FOLDER}/y_train.npy')\n    y_train_inp = np.pad(y_train, [[0,0],[1,0]], constant_values=SOS_TOKEN)\n    y_train_inp = y_train_inp[:,:N_TARGET_FRAMES]","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:05.580946Z","iopub.execute_input":"2023-08-18T20:20:05.581341Z","iopub.status.idle":"2023-08-18T20:20:05.591566Z","shell.execute_reply.started":"2023-08-18T20:20:05.581283Z","shell.execute_reply":"2023-08-18T20:20:05.590333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Batch","metadata":{"papermill":{"duration":0.030116,"end_time":"2023-07-03T18:22:00.685164","exception":false,"start_time":"2023-07-03T18:22:00.655048","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Example Batch For Debugging\nN_EXAMPLE_BATCH_SAMPLES = 1024\nN_EXAMPLE_BATCH_SAMPLES_SMALL = 32\nif USE_VAL:\n    # Example Batch\n    X_batch = {\n        'frames': np.copy(X_val[:N_EXAMPLE_BATCH_SAMPLES]) / DATASET_SCALAR,\n        'phrase': np.copy(y_val_inp[:N_EXAMPLE_BATCH_SAMPLES]),\n        'n_frames': np.copy(n_frames_val[:N_EXAMPLE_BATCH_SAMPLES]),\n    }\n    y_batch = np.copy(y_val[:N_EXAMPLE_BATCH_SAMPLES])\n    # Small Example Batch\n    X_batch_small = {\n        'frames': np.copy(X_val[:N_EXAMPLE_BATCH_SAMPLES_SMALL]) / DATASET_SCALAR,\n        'phrase': np.copy(y_val_inp[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n        'n_frames': np.copy(n_frames_val[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n    }\n    y_batch_small = np.copy(y_val[:N_EXAMPLE_BATCH_SAMPLES_SMALL])\nelse:\n        # Example Batch\n    X_batch = {\n        'frames': np.copy(X_train[:N_EXAMPLE_BATCH_SAMPLES]) / DATASET_SCALAR,\n        'phrase': np.copy(y_train_inp[:N_EXAMPLE_BATCH_SAMPLES]),\n        'n_frames': np.copy(n_frames_train[:N_EXAMPLE_BATCH_SAMPLES]),\n    }\n    y_batch = np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES])\n    # Small Example Batch\n    X_batch_small = {\n        'frames': np.copy(X_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]) / DATASET_SCALAR,\n        'phrase': np.copy(y_train_inp[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n        'n_frames': np.copy(n_frames_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n    }\n    y_batch_small = np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL])","metadata":{"papermill":{"duration":0.234215,"end_time":"2023-07-03T18:22:00.949655","exception":false,"start_time":"2023-07-03T18:22:00.71544","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:05.593713Z","iopub.execute_input":"2023-08-18T20:20:05.59412Z","iopub.status.idle":"2023-08-18T20:20:05.854922Z","shell.execute_reply.started":"2023-08-18T20:20:05.594086Z","shell.execute_reply":"2023-08-18T20:20:05.853887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Parquet","metadata":{"papermill":{"duration":0.031122,"end_time":"2023-07-03T18:22:01.012039","exception":false,"start_time":"2023-07-03T18:22:00.980917","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Read First Parquet File\n# example_parquet_df = pd.read_parquet(train['file_path'][0])\nexample_parquet_df = pd.read_parquet(INFERENCE_FILE_PATHS[0])\n\n# Each parquet file contains 1000 recordings\nprint(f'# Unique Recording: {example_parquet_df.index.nunique()}')\n# Display DataFrame layout\ndisplay(example_parquet_df.head())","metadata":{"papermill":{"duration":3.155142,"end_time":"2023-07-03T18:22:04.198581","exception":false,"start_time":"2023-07-03T18:22:01.043439","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:05.85635Z","iopub.execute_input":"2023-08-18T20:20:05.856812Z","iopub.status.idle":"2023-08-18T20:20:08.032531Z","shell.execute_reply.started":"2023-08-18T20:20:05.856767Z","shell.execute_reply":"2023-08-18T20:20:08.031518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Indices","metadata":{"papermill":{"duration":0.030745,"end_time":"2023-07-03T18:22:04.260976","exception":false,"start_time":"2023-07-03T18:22:04.230231","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Get indices in original dataframe\ndef get_idxs(df, words_pos, words_neg=[], ret_names=True, idxs_pos=None):\n    idxs = []\n    names = []\n    for w in words_pos:\n        for col_idx, col in enumerate(example_parquet_df.columns):\n            # Exclude Non Landmark Columns\n            if col in ['frame']:\n                continue\n                \n            col_idx = int(col.split('_')[-1])\n            # Check if column name contains all words\n            if (w in col) and (idxs_pos is None or col_idx in idxs_pos) and all([w not in col for w in words_neg]):\n                idxs.append(col_idx)\n                names.append(col)\n    # Convert to Numpy arrays\n    idxs = np.array(idxs)\n    names = np.array(names)\n    # Returns either both column indices and names\n    if ret_names:\n        return idxs, names\n    # Or only columns indices\n    else:\n        return idxs","metadata":{"papermill":{"duration":0.042953,"end_time":"2023-07-03T18:22:04.334786","exception":false,"start_time":"2023-07-03T18:22:04.291833","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.034022Z","iopub.execute_input":"2023-08-18T20:20:08.034811Z","iopub.status.idle":"2023-08-18T20:20:08.043242Z","shell.execute_reply.started":"2023-08-18T20:20:08.034778Z","shell.execute_reply":"2023-08-18T20:20:08.042135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lips Landmark Face Ids\nLIPS_LANDMARK_IDXS = np.array([\n    # Outer Lips\n#     61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146,\n#     76, 184, 74, 73, 72, 11, 302, 303, 304, 408, 306,307, 320, 404, 315, 16, 85, 180, 90, 77, \n    # Inner Lips\n    62, 183, 42, 41, 38, 12, 268, 271, 272, 407, 292, 325, 319, 403, 316, 15, 86, 179, 89, 96,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95,\n    ])\n# Pose Landmark Ids\nPOSE_LANDMARK_IDXS = np.array([\n    # Left Pose\n    12, 14, 16, 18, 20, 22,\n    # Right Pose\n    11, 13, 15, 17, 19, 21,\n    \n])\n# Nose Landmark Ids\nNOSE_LANDMARK_IDXS = np.array([\n    1, 2, 98, 327,\n])\n# Eye Landmark Ids\nEYES_LANDMARK_IDXS = np.array([\n    # Left Eye\n    263, 249, 390, 373, 374, 380, 381, 382, 362, 466, 388, 387, 386, 385, 384, 398,\n    # Right\n    33, 7, 163, 144, 145, 153, 154, 155, 133, 246, 161, 160, 159, 158, 157, 173,\n])\n# Landmark Indices for Left/Right hand without z axis in raw data\nLEFT_HAND_IDXS0, LEFT_HAND_NAMES0 = get_idxs(example_parquet_df, ['left_hand'], ['z'])\nRIGHT_HAND_IDXS0, RIGHT_HAND_NAMES0 = get_idxs(example_parquet_df, ['right_hand'], ['z'])\nLIPS_IDXS0, LIPS_NAMES0 = get_idxs(example_parquet_df, ['face'], ['z'], idxs_pos=LIPS_LANDMARK_IDXS)\nPOSE_IDXS0, POSE_NAMES0 = get_idxs(example_parquet_df, ['pose'], ['z'], idxs_pos=POSE_LANDMARK_IDXS)\nNOSE_IDXS0, NOSE_NAMES0 = get_idxs(example_parquet_df, ['face'], ['z'], idxs_pos=NOSE_LANDMARK_IDXS)\nEYES_IDXS0, EYES_NAMES0 = get_idxs(example_parquet_df, ['face'], ['z'], idxs_pos=EYES_LANDMARK_IDXS)\n# Column Names\nCOLUMNS0 = np.concatenate((\n    LEFT_HAND_NAMES0, RIGHT_HAND_NAMES0, LIPS_NAMES0, POSE_NAMES0, NOSE_NAMES0, EYES_NAMES0,\n))\nN_COLS0 = len(COLUMNS0)\n# Only X/Y axes are used\nN_DIMS0 = 2\n\nprint(f'N_COLS0: {N_COLS0}')","metadata":{"papermill":{"duration":0.052637,"end_time":"2023-07-03T18:22:04.418228","exception":false,"start_time":"2023-07-03T18:22:04.365591","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.046115Z","iopub.execute_input":"2023-08-18T20:20:08.046578Z","iopub.status.idle":"2023-08-18T20:20:08.065507Z","shell.execute_reply.started":"2023-08-18T20:20:08.046547Z","shell.execute_reply":"2023-08-18T20:20:08.064363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Landmark Indices in subset of dataframe with only COLUMNS selected\nLEFT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, LEFT_HAND_NAMES0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, RIGHT_HAND_NAMES0)).squeeze()\nHAND_IDXS = tf.concat((LEFT_HAND_IDXS, RIGHT_HAND_IDXS), axis=0)\nLIPS_IDXS = np.argwhere(np.isin(COLUMNS0, LIPS_NAMES0)).squeeze()\nPOSE_IDXS = np.argwhere(np.isin(COLUMNS0, POSE_NAMES0)).squeeze()\nNOSE_IDXS = np.argwhere(np.isin(COLUMNS0, NOSE_NAMES0)).squeeze()\nEYES_IDXS = np.argwhere(np.isin(COLUMNS0, EYES_NAMES0)).squeeze()\n# Landmark Indices in subset of dataframe with gradients\nHAND_IDXS_GRAD = tf.concat((HAND_IDXS, HAND_IDXS+N_COLS0), axis=0)\nLIPS_IDXS_GRAD = tf.concat((LIPS_IDXS, HAND_IDXS+N_COLS0), axis=0)\nPOSE_IDXS_GRAD = tf.concat((POSE_IDXS, HAND_IDXS+N_COLS0), axis=0)\nNOSE_IDXS_GRAD = tf.concat((NOSE_IDXS, HAND_IDXS+N_COLS0), axis=0)\nEYES_IDXS_GRAD = tf.concat((EYES_IDXS, HAND_IDXS+N_COLS0), axis=0)\n# Number of Columns\nN_COLS = N_COLS0\n# Only X/Y axes are used\nN_DIMS = 2\n\nprint(f'N_COLS: {N_COLS}')","metadata":{"papermill":{"duration":0.043034,"end_time":"2023-07-03T18:22:04.49226","exception":false,"start_time":"2023-07-03T18:22:04.449226","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.067136Z","iopub.execute_input":"2023-08-18T20:20:08.067614Z","iopub.status.idle":"2023-08-18T20:20:08.103097Z","shell.execute_reply.started":"2023-08-18T20:20:08.067583Z","shell.execute_reply":"2023-08-18T20:20:08.102134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Indices in processed data by axes with only dominant hand\nHAND_X_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'x' in name]\n    ).squeeze()\nHAND_Y_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'y' in name]\n    ).squeeze()\n# Names in processed data by axes\nHAND_X_NAMES = LEFT_HAND_NAMES0[HAND_X_IDXS]\nHAND_Y_NAMES = LEFT_HAND_NAMES0[HAND_Y_IDXS]","metadata":{"papermill":{"duration":0.040385,"end_time":"2023-07-03T18:22:04.56379","exception":false,"start_time":"2023-07-03T18:22:04.523405","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.104225Z","iopub.execute_input":"2023-08-18T20:20:08.104551Z","iopub.status.idle":"2023-08-18T20:20:08.111192Z","shell.execute_reply.started":"2023-08-18T20:20:08.10452Z","shell.execute_reply":"2023-08-18T20:20:08.109644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mean/STD Loading","metadata":{"papermill":{"duration":0.030739,"end_time":"2023-07-03T18:22:04.625417","exception":false,"start_time":"2023-07-03T18:22:04.594678","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Mean/Standard Deviations of data used for normalizing\nMEANS = np.load(f'{DATASET_FOLDER}/MEANS.npy').reshape(-1) / DATASET_SCALAR\nSTDS = np.load(f'{DATASET_FOLDER}/STDS.npy').reshape(-1) / DATASET_SCALAR\n\nprint(f'MEANS shape: {MEANS.shape}, STDS shape: {STDS.shape}')","metadata":{"papermill":{"duration":0.055053,"end_time":"2023-07-03T18:22:04.711522","exception":false,"start_time":"2023-07-03T18:22:04.656469","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.112678Z","iopub.execute_input":"2023-08-18T20:20:08.113302Z","iopub.status.idle":"2023-08-18T20:20:08.134549Z","shell.execute_reply.started":"2023-08-18T20:20:08.11327Z","shell.execute_reply":"2023-08-18T20:20:08.13365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorflow Preprocessing Layer","metadata":{"papermill":{"duration":0.032228,"end_time":"2023-07-03T18:22:04.775896","exception":false,"start_time":"2023-07-03T18:22:04.743668","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/hoyso48/1st-place-solution-training\ndef tf_nan_mean(x, axis=0, keepdims=False):\n    return tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis, keepdims=keepdims) / tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis, keepdims=keepdims)\n\ndef tf_nan_std(x, center=None, axis=0, keepdims=False):\n    if center is None:\n        center = tf_nan_mean(x, axis=axis,  keepdims=True)\n    d = x - center\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis, keepdims=keepdims))","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.135925Z","iopub.execute_input":"2023-08-18T20:20:08.136305Z","iopub.status.idle":"2023-08-18T20:20:08.143564Z","shell.execute_reply.started":"2023-08-18T20:20:08.136245Z","shell.execute_reply":"2023-08-18T20:20:08.1425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n    Tensorflow layer to process data in TFLite\n    Data needs to be processed in the model itself, so we can not use Python\n\"\"\" \nclass PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n        self.normalisation_correction = tf.constant(\n                    # Add 0.50 to x coordinates of left hand (original right hand) and substract 0.50 of right hand (original left hand)\n                     [0.50 if 'x_left_hand' in name else 0.00 for name in COLUMNS0],\n                dtype=tf.float32,\n            )\n    \n    @tf.function(\n        input_signature=[tf.TensorSpec(shape=[None,N_COLS0], dtype=tf.float32)],\n    )\n    def call(self, data0):\n        # Hacky\n        data = data0[None]\n        \n        # Fill NaN\n        data = tf.where(tf.math.is_nan(data), tf.constant(0.0, data.dtype), data)\n\n        # Squeeze Batch Dimension\n        data = tf.squeeze(data, axis=[0])\n        \n        # Crop/Pad\n        N_FRAMES = len(data)\n        data = tf.slice(\n            data,\n            [0,0],\n            [tf.math.minimum(N_FRAMES, N_TARGET_FRAMES), N_COLS],\n        )\n        \n        data = tf.pad(\n            data,\n            [[0,tf.math.maximum(0, N_TARGET_FRAMES-N_FRAMES)], [0,0]],\n        )\n        \n        # Explicit Shape\n        data = tf.reshape(data, [N_TARGET_FRAMES, N_COLS])\n        \n        return data\n    \npreprocess_layer = PreprocessLayer()","metadata":{"papermill":{"duration":0.056884,"end_time":"2023-07-03T18:22:04.865085","exception":false,"start_time":"2023-07-03T18:22:04.808201","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.145148Z","iopub.execute_input":"2023-08-18T20:20:08.145504Z","iopub.status.idle":"2023-08-18T20:20:08.164205Z","shell.execute_reply.started":"2023-08-18T20:20:08.145473Z","shell.execute_reply":"2023-08-18T20:20:08.16337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function To Test Preprocessing Layer\ndef test_preprocess_layer():\n    demo_sequence_id = example_parquet_df.index.unique()[1]\n    print(f'demo_sequence_id: {demo_sequence_id}')\n    demo_raw_data = example_parquet_df.loc[demo_sequence_id, COLUMNS0].values\n    if demo_raw_data.ndim == 1:\n        demo_raw_data = np.expand_dims(demo_raw_data, axis=0)\n    data = preprocess_layer(demo_raw_data)\n    print(f'demo_raw_data shape: {demo_raw_data.shape}')\n    print(f'data shape: {data.shape}')\n\n    return data\n\ndata = test_preprocess_layer()","metadata":{"papermill":{"duration":0.041278,"end_time":"2023-07-03T18:22:04.93815","exception":false,"start_time":"2023-07-03T18:22:04.896872","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.165901Z","iopub.execute_input":"2023-08-18T20:20:08.166248Z","iopub.status.idle":"2023-08-18T20:20:08.406818Z","shell.execute_reply.started":"2023-08-18T20:20:08.166217Z","shell.execute_reply":"2023-08-18T20:20:08.405873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assert Preprocessing Equals Training Data\nassert np.isclose(\n    data,\n    X_val[0].astype(np.float32) / DATASET_SCALAR,\n    rtol=1e-3,\n    atol=1e-3,\n).all()","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.409015Z","iopub.execute_input":"2023-08-18T20:20:08.409714Z","iopub.status.idle":"2023-08-18T20:20:08.419366Z","shell.execute_reply.started":"2023-08-18T20:20:08.409679Z","shell.execute_reply":"2023-08-18T20:20:08.418474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Dataset","metadata":{"papermill":{"duration":0.032173,"end_time":"2023-07-03T18:22:05.226814","exception":false,"start_time":"2023-07-03T18:22:05.194641","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Train Dataset Iterator\ndef get_train_dataset(X, y_input, y, n_frames, batch_size=BATCH_SIZE):\n    sample_idxs = np.arange(len(X))\n    while True:\n        # Get random indices\n        random_sample_idxs = np.random.choice(sample_idxs, batch_size)\n        \n        inputs = {\n            'frames': X[random_sample_idxs].astype(np.float32) / DATASET_SCALAR,\n            'phrase': y_input[random_sample_idxs],\n            'n_frames': n_frames[random_sample_idxs],\n        }\n        outputs = y[random_sample_idxs]\n        \n        yield inputs, outputs","metadata":{"papermill":{"duration":13.142335,"end_time":"2023-07-03T18:22:18.401146","exception":false,"start_time":"2023-07-03T18:22:05.258811","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.420912Z","iopub.execute_input":"2023-08-18T20:20:08.421479Z","iopub.status.idle":"2023-08-18T20:20:08.429878Z","shell.execute_reply.started":"2023-08-18T20:20:08.421446Z","shell.execute_reply":"2023-08-18T20:20:08.428978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Dataset\nif not USE_VAL:\n    train_dataset = get_train_dataset(X_train, y_train_inp, y_train, n_frames_train)","metadata":{"papermill":{"duration":0.03943,"end_time":"2023-07-03T18:22:18.471999","exception":false,"start_time":"2023-07-03T18:22:18.432569","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.432913Z","iopub.execute_input":"2023-08-18T20:20:08.433783Z","iopub.status.idle":"2023-08-18T20:20:08.439393Z","shell.execute_reply.started":"2023-08-18T20:20:08.433752Z","shell.execute_reply":"2023-08-18T20:20:08.438487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Steps Per Epoch\nif not USE_VAL:\n    TRAIN_STEPS_PER_EPOCH = math.ceil(N_TRAIN_SAMPLES / BATCH_SIZE)\n    print(f'TRAIN_STEPS_PER_EPOCH: {TRAIN_STEPS_PER_EPOCH}')","metadata":{"papermill":{"duration":38.171881,"end_time":"2023-07-03T18:22:56.675018","exception":false,"start_time":"2023-07-03T18:22:18.503137","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.441108Z","iopub.execute_input":"2023-08-18T20:20:08.442035Z","iopub.status.idle":"2023-08-18T20:20:08.449408Z","shell.execute_reply.started":"2023-08-18T20:20:08.441984Z","shell.execute_reply":"2023-08-18T20:20:08.448448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_VAL:\n    X_batch_train, y_batch_train = next(train_dataset)\n\n    for k, v in X_batch_train.items():\n        print(f'{k} shape: {v.shape}, min: {v.min():.2f}, max: {v.max():.2f}')","metadata":{"papermill":{"duration":19.706209,"end_time":"2023-07-03T18:23:16.577086","exception":false,"start_time":"2023-07-03T18:22:56.870877","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.460342Z","iopub.execute_input":"2023-08-18T20:20:08.4606Z","iopub.status.idle":"2023-08-18T20:20:08.465914Z","shell.execute_reply.started":"2023-08-18T20:20:08.460578Z","shell.execute_reply":"2023-08-18T20:20:08.464681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation Dataset","metadata":{"papermill":{"duration":0.189898,"end_time":"2023-07-03T18:23:16.958164","exception":false,"start_time":"2023-07-03T18:23:16.768266","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Validation Set\ndef get_val_dataset(X, y_input, y, n_frames, batch_size=BATCH_SIZE):\n    offsets = np.arange(0, len(X), batch_size)\n    while True:\n        # Iterate over whole validation set\n        for offset in offsets:\n            inputs = {\n                'frames': X[offset:offset+batch_size].astype(np.float32) / DATASET_SCALAR,\n                'phrase': y_input[offset:offset+batch_size],\n                'n_frames': n_frames[offset:offset+batch_size],\n            }\n            outputs = y[offset:offset+batch_size]\n\n            yield inputs, outputs","metadata":{"papermill":{"duration":0.198274,"end_time":"2023-07-03T18:23:17.346494","exception":false,"start_time":"2023-07-03T18:23:17.14822","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.467503Z","iopub.execute_input":"2023-08-18T20:20:08.468045Z","iopub.status.idle":"2023-08-18T20:20:08.476798Z","shell.execute_reply.started":"2023-08-18T20:20:08.46801Z","shell.execute_reply":"2023-08-18T20:20:08.475373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation Dataset\nif USE_VAL:\n    val_dataset = get_val_dataset(X_val, y_val_inp, y_val, n_frames_val)","metadata":{"papermill":{"duration":0.196927,"end_time":"2023-07-03T18:23:17.731909","exception":false,"start_time":"2023-07-03T18:23:17.534982","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.478429Z","iopub.execute_input":"2023-08-18T20:20:08.478838Z","iopub.status.idle":"2023-08-18T20:20:08.488908Z","shell.execute_reply.started":"2023-08-18T20:20:08.478806Z","shell.execute_reply":"2023-08-18T20:20:08.487839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    N_VAL_STEPS_PER_EPOCH = math.ceil(N_VAL_SAMPLES / BATCH_SIZE)\n    print(f'N_VAL_STEPS_PER_EPOCH: {N_VAL_STEPS_PER_EPOCH}')","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.492089Z","iopub.execute_input":"2023-08-18T20:20:08.492397Z","iopub.status.idle":"2023-08-18T20:20:08.500189Z","shell.execute_reply.started":"2023-08-18T20:20:08.492362Z","shell.execute_reply":"2023-08-18T20:20:08.499163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    X_batch_val, y_batch_val = next(val_dataset)\n\n    for k, v in X_batch_val.items():\n        print(f'{k} shape: {v.shape}, min: {v.min():.2f}, max: {v.max():.2f}')\n    \n    print(f'y_batch_val shape: {y_batch_val.shape}, dtype: {y_batch_val.dtype}')","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.501772Z","iopub.execute_input":"2023-08-18T20:20:08.50223Z","iopub.status.idle":"2023-08-18T20:20:08.517899Z","shell.execute_reply.started":"2023-08-18T20:20:08.5022Z","shell.execute_reply":"2023-08-18T20:20:08.516727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Config","metadata":{}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# final embedding and transformer embedding size\nUNITS_ENCODER = 320\nUNITS_DECODER = 160\n\n# Transformer\nNUM_BLOCKS_ENCODER = 4\nNUM_BLOCKS_DECODER = 2\nNUM_HEADS = 4\nMLP_RATIO = 2\n\n# Dropout\nif DATASET == 'competition':\n    INPUT_DROPOUT_RATIO = 0.10\n    EMBEDDING_DROPOUT = 0.00\n    MHA_DROPOUT_RATIO = 0.20\n    MLP_DROPOUT_RATIO = 0.30\n    CLASSIFIER_DROPOUT_RATIO = 0.10\nelif DATASET == 'supplemental':\n    INPUT_DROPOUT_RATIO = 0.00\n    EMBEDDING_DROPOUT = 0.00\n    MHA_DROPOUT_RATIO = 0.00\n    MLP_DROPOUT_RATIO = 0.30\n    CLASSIFIER_DROPOUT_RATIO = 0.00\n    \n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.520023Z","iopub.execute_input":"2023-08-18T20:20:08.520917Z","iopub.status.idle":"2023-08-18T20:20:08.528258Z","shell.execute_reply.started":"2023-08-18T20:20:08.520887Z","shell.execute_reply":"2023-08-18T20:20:08.527443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Embedding","metadata":{"papermill":{"duration":0.187359,"end_time":"2023-07-03T18:23:24.675248","exception":false,"start_time":"2023-07-03T18:23:24.487889","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n        self.supports_masking = True\n        \n    def build(self, input_shape):\n        # Embedding for missing landmark in frame, initizlied with zeros\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # Embedding\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n            tf.keras.layers.BatchNormalization(momentum=0.95, name=f'{self.name}_bn_1'),\n            tf.keras.layers.Activation(activation=GELU, name=f'{self.name}_gelu_1'),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x, training=False):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.dense(x),\n            )","metadata":{"papermill":{"duration":0.205674,"end_time":"2023-07-03T18:23:25.067269","exception":false,"start_time":"2023-07-03T18:23:24.861595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.529386Z","iopub.execute_input":"2023-08-18T20:20:08.530308Z","iopub.status.idle":"2023-08-18T20:20:08.540773Z","shell.execute_reply.started":"2023-08-18T20:20:08.530278Z","shell.execute_reply":"2023-08-18T20:20:08.539735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Embedding","metadata":{}},{"cell_type":"code","source":"# Creates embedding for each frame\nclass Embedding(tf.keras.Model):\n    def __init__(self):\n        super(Embedding, self).__init__()\n        self.supports_masking = True\n\n    def build(self, input_shape):\n        # Positional embedding for each frame index\n        self.positional_embedding = tf.Variable(\n            initial_value=tf.zeros([N_TARGET_FRAMES, UNITS_ENCODER], dtype=tf.float32),\n            trainable=True,\n            name='embedding_positional_encoder',\n        )\n        # Embedding layer for Landmarks\n        self.hands_embedding = LandmarkEmbedding(UNITS_ENCODER, 'hands')\n        self.lips_embedding = LandmarkEmbedding(UNITS_ENCODER, 'lips')\n        self.pose_embedding = LandmarkEmbedding(UNITS_ENCODER, 'pose')\n        self.nose_embedding = LandmarkEmbedding(UNITS_ENCODER, 'nose')\n        self.eyes_embedding = LandmarkEmbedding(UNITS_ENCODER, 'eyes')\n        # Landmark Weights\n        self.landmark_weights = tf.Variable(tf.zeros([5], dtype=tf.float32), name='landmark_weights')\n\n    def call(self, x, training=False):\n        # Add Gradient\n        x_grad = tf.cond(\n            tf.shape(x)[1] > 1,\n            lambda : tf.pad(x[:,1:] - x[:,:-1], [[0,0],[0,1],[0,0]]),\n            lambda: tf.zeros_like(x),\n        )\n        x = tf.concat((x, x_grad), axis=2)\n         # Replace Missing Values With Means\n        x = tf.where(x == 0.0, MEANS, x)\n        # Normalize\n        x = tf.where(\n            tf.math.equal(x, 0.0),\n            0.0,\n            (x - MEANS) / STDS,\n        )\n        # Embeddings\n        x = x = tf.stack((\n            self.hands_embedding(tf.gather(x, HAND_IDXS_GRAD, axis=2)),\n            self.lips_embedding(tf.gather(x, LIPS_IDXS_GRAD, axis=2)),\n            self.pose_embedding(tf.gather(x, POSE_IDXS_GRAD, axis=2)),\n            self.nose_embedding(tf.gather(x, NOSE_IDXS_GRAD, axis=2)),\n            self.eyes_embedding(tf.gather(x, EYES_IDXS_GRAD, axis=2)),\n        ), axis=3)\n        # Weight Embeddings\n        x = x * tf.nn.softmax(self.landmark_weights)\n        # Sum Embeddings\n        x = tf.reduce_sum(x, axis=3)\n        # Add Positional Encoding\n        x = x + self.positional_embedding\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.543461Z","iopub.execute_input":"2023-08-18T20:20:08.543839Z","iopub.status.idle":"2023-08-18T20:20:08.559376Z","shell.execute_reply.started":"2023-08-18T20:20:08.543801Z","shell.execute_reply":"2023-08-18T20:20:08.558538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer","metadata":{}},{"cell_type":"code","source":"# based on: https://stackoverflow.com/questions/67342988/verifying-the-implementation-of-multihead-attention-in-transformer\n# replaced softmax with softmax layer to support masked softmax\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self, d_model, n_heads, dropout, name, d_out=None, padding='SAME'):\n        super(MultiHeadAttention,self).__init__(name=f'multi_head_attention_{name}')\n        self.d_model = d_model\n        self.n_heads = n_heads\n        self.depth = d_model // 2\n        self.scale = 1.0 / tf.math.sqrt(self.depth * 1.0)\n        self.wqkv = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.depth * 3, use_bias=False),\n            tf.keras.layers.Reshape([N_TARGET_FRAMES, self.n_heads, self.depth // self.n_heads * 3]),\n            tf.keras.layers.Permute([2, 1, 3]),\n        ])\n        self.wo = tf.keras.layers.Dense(d_model if d_out is None else d_out, use_bias=False)\n        self.reshape = tf.keras.Sequential([\n            tf.keras.layers.Permute([2, 1, 3]),\n            tf.keras.layers.Reshape([N_TARGET_FRAMES, self.depth]),\n        ])\n        self.do = tf.keras.layers.Dropout(dropout)\n        self.supports_masking = True\n        self.focus_factor = 3.0\n        self.kernel_function = tf.nn.relu\n        self.dwc = tf.keras.Sequential([\n            tf.keras.layers.Permute([2, 1, 3]),\n            tf.keras.layers.Reshape([N_TARGET_FRAMES, self.depth]),\n            tf.keras.layers.Conv1D(self.depth, kernel_size=11, padding=padding, use_bias=False),\n        ])\n        self.scale = tf.Variable(\n            initial_value=tf.zeros([self.n_heads, 1, self.depth // self.n_heads]),\n            trainable=True,\n            name=f'multi_head_attention_{name}_scale'\n        )\n        \n    def call(self, x, attention_mask, training=False):\n        QKV = self.wqkv(x)\n        Q, K, V = tf.split(QKV, [self.depth // self.n_heads] * 3, axis=-1)\n        Q = (self.kernel_function(Q) + EPS) / tf.math.softplus(self.scale)\n        K = (self.kernel_function(K) + EPS) / tf.math.softplus(self.scale)\n                \n        Q_norm = tf.norm(Q, axis=3, keepdims=True, ord='euclidean')\n        K_norm = tf.norm(Q, axis=3, keepdims=True, ord='euclidean')\n        Q_focus = tf.maximum(Q ** self.focus_factor, EPS)\n        K_focus = tf.maximum(K ** self.focus_factor, EPS)\n        Q = (Q_focus / tf.norm(Q_focus, axis=3, keepdims=True, ord='euclidean')) * Q_norm\n        K = (K_focus / tf.norm(K_focus, axis=3, keepdims=True, ord='euclidean')) * K_norm\n\n        attention_scores = tf.matmul(Q, K, transpose_b=True) * attention_mask\n        attention_scores = attention_scores / (tf.reduce_sum(attention_scores, axis=3, keepdims=True) + EPS)\n\n        x = attention_scores @ V\n        \n        x = self.reshape(x)\n        \n        x = x + self.dwc(V)\n        \n        x = self.wo(x)\n        \n        x = self.do(x, training=training)\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.56086Z","iopub.execute_input":"2023-08-18T20:20:08.561276Z","iopub.status.idle":"2023-08-18T20:20:08.579882Z","shell.execute_reply.started":"2023-08-18T20:20:08.561244Z","shell.execute_reply":"2023-08-18T20:20:08.579041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer Decoder","metadata":{}},{"cell_type":"code","source":"# based on: https://stackoverflow.com/questions/67342988/verifying-the-implementation-of-multihead-attention-in-transformer\n# replaced softmax with softmax layer to support masked softmax\nclass MultiHeadAttentionDecoder(tf.keras.layers.Layer):\n    def __init__(self, d_model, n_heads, dropout, name, d_out=None):\n        super(MultiHeadAttentionDecoder,self).__init__(name=f'multi_head_attention_decoder_{name}')\n        self.d_model = d_model\n        self.n_heads = n_heads\n        self.depth = d_model // 2\n        self.scale = 1.0 / tf.math.sqrt(self.depth * 1.0)\n        self.wq = self.fused_mha()\n        self.wo = tf.keras.layers.Dense(d_model if d_out is None else d_out, use_bias=False)\n        self.softmax = tf.keras.layers.Softmax()\n        self.reshape = tf.keras.Sequential([\n            tf.keras.layers.Permute([2, 1, 3]),\n            tf.keras.layers.Reshape([N_TARGET_FRAMES, self.depth]),\n        ])\n        self.do = tf.keras.layers.Dropout(dropout)\n        self.supports_masking = True\n        \n    def fused_mha(self):\n        return tf.keras.Sequential([\n            tf.keras.layers.Dense(self.depth, use_bias=False),\n            tf.keras.layers.Reshape([N_TARGET_FRAMES, self.n_heads, self.depth // self.n_heads]),\n            tf.keras.layers.Permute([2, 1, 3]),\n        ])\n        \n    def call(self, q, K, V, attention_mask=None, training=False):\n        Q = self.wq(q)\n        x = tf.matmul(Q, K, transpose_b=True) * self.scale\n        x = self.softmax(x, mask=attention_mask) @ V\n        x = self.reshape(x)\n        x = self.wo(x)\n        x = self.do(x, training=training)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.583243Z","iopub.execute_input":"2023-08-18T20:20:08.583532Z","iopub.status.idle":"2023-08-18T20:20:08.596747Z","shell.execute_reply.started":"2023-08-18T20:20:08.58351Z","shell.execute_reply":"2023-08-18T20:20:08.596019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encoder","metadata":{}},{"cell_type":"code","source":"# Encoder based on multiple transformer blocks\nclass Encoder(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Encoder, self).__init__(name='encoder')\n        self.num_blocks = num_blocks\n        self.supports_masking = True\n    \n    def build(self, input_shape):\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS, name=f'encoder_ln_{i}a'))\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS_ENCODER, NUM_HEADS, MHA_DROPOUT_RATIO, i))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS, name=f'encoder_ln_{i}b'))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS_ENCODER * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS_ENCODER, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),\n            ]))\n        # Optional Projection to Decoder Dimension\n        if UNITS_ENCODER != UNITS_DECODER:\n            self.dense_out = tf.keras.layers.Dense(UNITS_DECODER, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False)\n            self.apply_dense_out = True\n        else:\n            self.apply_dense_out = False\n           \n        self.decoder_wks = []\n        self.decoder_wvs = []\n        for i in range(NUM_BLOCKS_DECODER):\n            self.decoder_wks.append(self.fused_mha())\n            self.decoder_wvs.append(self.fused_mha())\n            \n    def fused_mha(self):\n        return tf.keras.Sequential([\n            tf.keras.layers.Dense(UNITS_DECODER // 2, use_bias=False),\n            tf.keras.layers.Reshape([N_TARGET_FRAMES, NUM_HEADS, UNITS_DECODER // NUM_HEADS // 2]),\n            tf.keras.layers.Permute([2, 1, 3]),\n        ])\n    \n    def get_attention_mask(self, x_inp):\n        # Attention Mask\n        attention_mask = tf.math.count_nonzero(x_inp, axis=[2], keepdims=True, dtype=tf.int32)\n        attention_mask = tf.math.count_nonzero(attention_mask, axis=[2], keepdims=False, dtype=tf.float32)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        return attention_mask\n        \n    def call(self, x, x_inp, training=False):\n        attention_mask = self.get_attention_mask(x_inp)\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x = ln_1(x + mha(x, attention_mask))\n            x = ln_2(x + mlp(x))\n            \n        # Optional Projection to Decoder Dimension\n        if self.apply_dense_out:\n            x = self.dense_out(x)\n            \n        decoder_wks = []\n        decoder_wvs = []\n        for mha_wks, mha_wvs in zip(self.decoder_wks, self.decoder_wvs):\n            decoder_wks.append(mha_wks(x))\n            decoder_wvs.append(mha_wvs(x))\n            \n        return x, attention_mask, decoder_wks, decoder_wvs","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.598043Z","iopub.execute_input":"2023-08-18T20:20:08.599012Z","iopub.status.idle":"2023-08-18T20:20:08.619059Z","shell.execute_reply.started":"2023-08-18T20:20:08.598964Z","shell.execute_reply":"2023-08-18T20:20:08.61804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Decoder","metadata":{}},{"cell_type":"code","source":"# Decoder based on multiple transformer blocks\nclass Decoder(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Decoder, self).__init__(name='decoder')\n        self.num_blocks = num_blocks\n        self.supports_masking = True\n    \n    def build(self, input_shape):\n        # Causal Mask Batch Size 1\n        self.causal_mask = self.get_causal_attention_mask()\n        # Positional Embedding, initialized with zeros\n        self.positional_embedding = tf.Variable(\n            initial_value=tf.zeros([N_TARGET_FRAMES, UNITS_DECODER]),\n            trainable=True,\n            name='decoder_positional_encoder',\n        )\n        # Character Embedding\n        self.char_emb = tf.keras.layers.Embedding(\n            input_dim=N_UNIQUE_CHARACTERS,\n            output_dim=UNITS_DECODER,\n            embeddings_initializer='zeros',\n        )\n        # Positional Encoder MHA\n        self.phrase_emb_mha = MultiHeadAttention(UNITS_DECODER, NUM_HEADS, MHA_DROPOUT_RATIO, 'phrase', padding='CAUSAL')\n        self.phrase_emb_ln = tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS, name=f'decoder_ln_phrase')\n        # First Layer Normalisation\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS, name=f'decoder_ln_{i}a'))\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttentionDecoder(UNITS_DECODER, NUM_HEADS, MHA_DROPOUT_RATIO, i))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS, name=f'decoder_ln_{i}b'))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS_DECODER * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS_DECODER, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),\n            ]))\n            \n    def get_causal_attention_mask(self):\n        i = tf.range(N_TARGET_FRAMES)\n        i = tf.expand_dims(i, axis=1)\n        j = tf.range(N_TARGET_FRAMES)\n        mask = tf.cast(i >= j, dtype=tf.float32)\n        mask = tf.expand_dims(mask, axis=0)\n        mask = tf.expand_dims(mask, axis=0)\n        return mask\n    \n    def call(self, encoder_outputs, phrase, x_inp, attention_mask, decoder_wks, decoder_wvs, training=False):\n        # Character Embedding\n        x = self.char_emb(phrase)\n        # Add Positional Embedding\n        x = x + self.positional_embedding\n        # Causal Attention\n        x = self.phrase_emb_ln(x + self.phrase_emb_mha(x, attention_mask=self.causal_mask, training=training))\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp, K, V in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps, decoder_wks, decoder_wvs):\n            x = ln_1(x + mha(x, K, V, attention_mask=attention_mask, training=training))\n            x = ln_2(x + mlp(x))\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.622471Z","iopub.execute_input":"2023-08-18T20:20:08.62275Z","iopub.status.idle":"2023-08-18T20:20:08.639781Z","shell.execute_reply.started":"2023-08-18T20:20:08.622728Z","shell.execute_reply":"2023-08-18T20:20:08.638859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Non Pad/SOS/EOS Token Accuracy","metadata":{}},{"cell_type":"code","source":"# TopK accuracy for multi dimensional output\nclass TopKAccuracy(tf.keras.metrics.Metric):\n    def __init__(self, k, **kwargs):\n        super(TopKAccuracy, self).__init__(name=f'top{k}acc', **kwargs)\n        self.top_k_acc = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=k)\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        y_true = tf.reshape(y_true, [-1])\n        y_pred = tf.reshape(y_pred, [-1, N_UNIQUE_CHARACTERS])\n        character_idxs = tf.where(y_true < N_UNIQUE_CHARACTERS0)\n        y_true = tf.gather(y_true, character_idxs, axis=0)\n        y_pred = tf.gather(y_pred, character_idxs, axis=0)\n        self.top_k_acc.update_state(y_true, y_pred)\n\n    def result(self):\n        return self.top_k_acc.result()\n    \n    def reset_state(self):\n        self.top_k_acc.reset_state()","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.641173Z","iopub.execute_input":"2023-08-18T20:20:08.641531Z","iopub.status.idle":"2023-08-18T20:20:08.653527Z","shell.execute_reply.started":"2023-08-18T20:20:08.641501Z","shell.execute_reply":"2023-08-18T20:20:08.652741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sparse Categorical Crossentropy With Label Smoothing","metadata":{}},{"cell_type":"code","source":"# source:: https://stackoverflow.com/questions/60689185/label-smoothing-for-sparse-categorical-crossentropy\ndef scce_with_ls(y_true, y_pred):\n    # Filter Pad Tokens\n    idxs = tf.where(y_true != PAD_TOKEN)\n    y_true = tf.gather_nd(y_true, idxs)\n    y_pred = tf.gather_nd(y_pred, idxs)\n    # One Hot Encode Sparsely Encoded Target Sign\n    y_true = tf.cast(y_true, tf.int32)\n    y_true = tf.one_hot(y_true, N_UNIQUE_CHARACTERS, axis=1)\n    # Categorical Crossentropy with native label smoothing support\n    loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, label_smoothing=0.25, from_logits=True)\n    loss = tf.math.reduce_mean(loss)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.654938Z","iopub.execute_input":"2023-08-18T20:20:08.655289Z","iopub.status.idle":"2023-08-18T20:20:08.664599Z","shell.execute_reply.started":"2023-08-18T20:20:08.655255Z","shell.execute_reply":"2023-08-18T20:20:08.663391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def get_model(load_weights=False):\n    with strategy.scope():\n        # Inputs\n        frames_inp = tf.keras.layers.Input([N_TARGET_FRAMES, N_COLS], dtype=tf.float32, name='frames')\n        phrase_inp = tf.keras.layers.Input([MAX_PHRASE_LENGTH], dtype=tf.int32, name='phrase')\n        # Frames\n        x = frames_inp\n\n        # Masking\n        x = tf.keras.layers.Masking(mask_value=0.0, input_shape=[N_TARGET_FRAMES, N_COLS])(x)\n        phrase = tf.keras.layers.Masking(mask_value=PAD_TOKEN, input_shape=[MAX_PHRASE_LENGTH])(phrase_inp)\n\n        # Embedding\n        x = Embedding()(x)\n\n        # Encoder Transformer Blocks\n        x, attention_mask, decoder_wks, decoder_wvs = Encoder(NUM_BLOCKS_ENCODER)(x, frames_inp)\n\n        # Decoder\n        x = Decoder(NUM_BLOCKS_DECODER)(x, phrase, frames_inp, attention_mask, decoder_wks, decoder_wvs)\n        \n        # Classifier\n        x = tf.keras.Sequential([\n            # Dropout\n            tf.keras.layers.Dropout(CLASSIFIER_DROPOUT_RATIO),\n            # Output Neurons\n            tf.keras.layers.Dense(N_UNIQUE_CHARACTERS, activation=tf.keras.activations.linear, kernel_initializer=INIT_GLOROT_UNIFORM, use_bias=False),\n        ], name='classifier')(x)\n\n        outputs = x\n\n        # Create Tensorflow Model\n        model = tf.keras.models.Model(inputs=[frames_inp, phrase_inp], outputs=outputs)\n        \n        # Load Weights\n        model.load_weights('/kaggle/input/aslfr-training-tpu-dataset/model.h5')\n\n        # Set Untrainable\n        model.trainable = False\n        \n        # Compile Model\n        model.compile(loss=scce_with_ls)\n\n        return model","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:08.666099Z","iopub.execute_input":"2023-08-18T20:20:08.666392Z","iopub.status.idle":"2023-08-18T20:20:08.678487Z","shell.execute_reply.started":"2023-08-18T20:20:08.666368Z","shell.execute_reply":"2023-08-18T20:20:08.677869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Input data\nfor k, v in X_batch.items():\n    print(f'{k}: {v.shape}')","metadata":{"papermill":{"duration":0.196124,"end_time":"2023-07-03T18:23:25.450453","exception":false,"start_time":"2023-07-03T18:23:25.254329","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.679723Z","iopub.execute_input":"2023-08-18T20:20:08.680513Z","iopub.status.idle":"2023-08-18T20:20:08.692768Z","shell.execute_reply.started":"2023-08-18T20:20:08.680482Z","shell.execute_reply":"2023-08-18T20:20:08.691734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = get_model()","metadata":{"papermill":{"duration":7.395358,"end_time":"2023-07-03T18:23:33.033166","exception":false,"start_time":"2023-07-03T18:23:25.637808","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:08.69424Z","iopub.execute_input":"2023-08-18T20:20:08.694697Z","iopub.status.idle":"2023-08-18T20:20:13.054089Z","shell.execute_reply.started":"2023-08-18T20:20:08.694633Z","shell.execute_reply":"2023-08-18T20:20:13.053102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary(expand_nested=True)","metadata":{"papermill":{"duration":0.348138,"end_time":"2023-07-03T18:23:33.580487","exception":false,"start_time":"2023-07-03T18:23:33.232349","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:13.056933Z","iopub.execute_input":"2023-08-18T20:20:13.057657Z","iopub.status.idle":"2023-08-18T20:20:13.266628Z","shell.execute_reply.started":"2023-08-18T20:20:13.057622Z","shell.execute_reply":"2023-08-18T20:20:13.265877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Model Architecture\ntf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True, show_layer_activations=True)","metadata":{"papermill":{"duration":0.880455,"end_time":"2023-07-03T18:23:34.672087","exception":false,"start_time":"2023-07-03T18:23:33.791632","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:13.267676Z","iopub.execute_input":"2023-08-18T20:20:13.268053Z","iopub.status.idle":"2023-08-18T20:20:13.676055Z","shell.execute_reply.started":"2023-08-18T20:20:13.268019Z","shell.execute_reply":"2023-08-18T20:20:13.675165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Verify Correct Training Flag","metadata":{"papermill":{"duration":0.226308,"end_time":"2023-07-03T18:23:35.121685","exception":false,"start_time":"2023-07-03T18:23:34.895377","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Verify static output for inference\ndef verify_correct_training_flag():\n    pred = model(X_batch_small, training=False)\n    for _ in tqdm(range(10), disable=DISABLE_TQDM):\n        assert tf.reduce_min(tf.cast(pred == model(X_batch_small, training=False), tf.int8)) == 1\n\n    # Verify at least 99% varying output due to dropout during training\n    for _ in tqdm(range(10), disable=DISABLE_TQDM):\n        assert tf.reduce_mean(tf.cast(pred != model(X_batch_small, training=True), tf.float32)) > 0.99\n        \nverify_correct_training_flag()","metadata":{"papermill":{"duration":15.268681,"end_time":"2023-07-03T18:23:50.600798","exception":false,"start_time":"2023-07-03T18:23:35.332117","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:13.677078Z","iopub.execute_input":"2023-08-18T20:20:13.677386Z","iopub.status.idle":"2023-08-18T20:20:22.662247Z","shell.execute_reply.started":"2023-08-18T20:20:13.677358Z","shell.execute_reply":"2023-08-18T20:20:22.661404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Decay Callback","metadata":{"papermill":{"duration":0.210012,"end_time":"2023-07-03T18:25:01.501951","exception":false,"start_time":"2023-07-03T18:25:01.291939","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"papermill":{"duration":0.219642,"end_time":"2023-07-03T18:25:01.929825","exception":false,"start_time":"2023-07-03T18:25:01.710183","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:22.66632Z","iopub.execute_input":"2023-08-18T20:20:22.668515Z","iopub.status.idle":"2023-08-18T20:20:22.677288Z","shell.execute_reply.started":"2023-08-18T20:20:22.668481Z","shell.execute_reply":"2023-08-18T20:20:22.675907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate Initialized Model","metadata":{"papermill":{"duration":0.210408,"end_time":"2023-07-03T18:25:02.349841","exception":false,"start_time":"2023-07-03T18:25:02.139433","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Evaluate Initialized Model On Validation Data\n_ = model.evaluate(\n    val_dataset if USE_VAL else train_dataset,\n    steps=N_VAL_STEPS_PER_EPOCH if USE_VAL else TRAIN_STEPS_PER_EPOCH,\n    verbose=VERBOSE,\n)","metadata":{"papermill":{"duration":51.128172,"end_time":"2023-07-03T18:25:53.690563","exception":false,"start_time":"2023-07-03T18:25:02.562391","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:22.681453Z","iopub.execute_input":"2023-08-18T20:20:22.68349Z","iopub.status.idle":"2023-08-18T20:20:40.997841Z","shell.execute_reply.started":"2023-08-18T20:20:22.683458Z","shell.execute_reply":"2023-08-18T20:20:40.996911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline","metadata":{"papermill":{"duration":0.257831,"end_time":"2023-07-03T18:25:54.198689","exception":false,"start_time":"2023-07-03T18:25:53.940858","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# baseline accuracy when only pad token is predicted\nif USE_VAL:\n    baseline_accuracy = np.mean(y_val == PAD_TOKEN)\nelse:\n    baseline_accuracy = np.mean(y_train == PAD_TOKEN)\nprint(f'Baseline Accuracy: {baseline_accuracy:.4f}')","metadata":{"papermill":{"duration":0.225857,"end_time":"2023-07-03T18:25:54.634573","exception":false,"start_time":"2023-07-03T18:25:54.408716","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-08-18T20:20:40.999452Z","iopub.execute_input":"2023-08-18T20:20:41.000307Z","iopub.status.idle":"2023-08-18T20:20:41.010142Z","shell.execute_reply.started":"2023-08-18T20:20:41.000269Z","shell.execute_reply":"2023-08-18T20:20:41.008918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Levenstein Distance","metadata":{}},{"cell_type":"code","source":"# Output Predictions to string\ndef outputs2phrase(outputs):\n    if outputs.ndim == 2:\n        outputs = np.argmax(outputs, axis=1)\n    \n    return ''.join([ORD2CHAR.get(s, '') for s in outputs])","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:41.011744Z","iopub.execute_input":"2023-08-18T20:20:41.012105Z","iopub.status.idle":"2023-08-18T20:20:41.018024Z","shell.execute_reply.started":"2023-08-18T20:20:41.012074Z","shell.execute_reply":"2023-08-18T20:20:41.017047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function(jit_compile=True)\ndef predict_phrase(frames):\n    # Add Batch Dimension\n    frames = tf.expand_dims(frames, axis=0)\n    frames = tf.cast(frames, tf.float32) / DATASET_SCALAR\n    # Start Phrase\n    phrase = tf.fill([1,MAX_PHRASE_LENGTH], PAD_TOKEN)\n    # Stop After EOS Token\n    stop = False\n    # Predict 1 Character At A Time\n    for idx in tf.range(MAX_PRED_LENGTH):\n        # Cast phrase to int8\n        phrase = tf.cast(phrase, tf.int8)\n        phrase = tf.pad(phrase, [[0,0], [1,0]], constant_values=SOS_TOKEN)\n        phrase = tf.slice(phrase, [0,0], [1,MAX_PHRASE_LENGTH])\n        # Predict Next Token\n        outputs = tf.cond(\n                stop,\n                lambda: tf.one_hot(tf.cast(phrase, tf.int32), N_UNIQUE_CHARACTERS),\n                lambda: model({ 'frames': frames, 'phrase': phrase }),\n            )\n        # Add predicted token to input phrase\n        phrase = tf.cast(phrase, tf.int32)\n        phrase = tf.where(\n            tf.range(MAX_PHRASE_LENGTH) < idx + 2,\n            tf.argmax(outputs, axis=2, output_type=tf.int32),\n            phrase,\n        )\n        # Predicted Token\n        predicted_token = phrase[0,idx]\n        # If EOS (End Of Sentence) token is predicted stop\n        if not stop:\n            stop = predicted_token == EOS_TOKEN\n\n    # Squeeze outputs\n    outputs = tf.squeeze(phrase, axis=0)\n    outputs = tf.one_hot(outputs, N_UNIQUE_CHARACTERS)\n\n    # Return a dictionary with the output tensor\n    return outputs","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:41.019476Z","iopub.execute_input":"2023-08-18T20:20:41.020184Z","iopub.status.idle":"2023-08-18T20:20:41.031886Z","shell.execute_reply.started":"2023-08-18T20:20:41.02015Z","shell.execute_reply":"2023-08-18T20:20:41.030774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Levenstein Distance Train","metadata":{}},{"cell_type":"code","source":"# Compute Levenstein Distances\ndef get_ld(X, y, N=1024):\n    LD_TRAIN = []\n    for idx, (frames, phrase_true) in enumerate(zip(tqdm(X, total=N), y)):\n        # Predict Phrase and Convert to String\n        phrase_pred = predict_phrase(frames).numpy()\n        phrase_pred = outputs2phrase(phrase_pred)\n        # True Phrase Ordinal to String\n        phrase_true = outputs2phrase(phrase_true)\n        # Add Levenstein Distance\n        LD_TRAIN.append({\n            'phrase_true': phrase_true,\n            'phrase_true_len': len(phrase_true),\n            'phrase_pred': phrase_pred,\n            'levenshtein_distance': levenshtein(phrase_pred, phrase_true),\n        })\n        # Take subset in interactive mode\n        if idx == N:\n            break\n            \n    # Convert to DataFrame\n    LD_TRAIN_DF = pd.DataFrame(LD_TRAIN)\n    \n    return LD_TRAIN_DF","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:41.033873Z","iopub.execute_input":"2023-08-18T20:20:41.034858Z","iopub.status.idle":"2023-08-18T20:20:41.045782Z","shell.execute_reply.started":"2023-08-18T20:20:41.034826Z","shell.execute_reply":"2023-08-18T20:20:41.044816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    LD_VAL_DF = get_ld(X_val, y_val, N=N_VAL_SAMPLES)\n\n    # Display Errors\n    display(LD_VAL_DF.sort_values('levenshtein_distance', ascending=False).head(100))\nelse:\n    LD_VAL_DF = get_ld(X_train, y_train)\n\n    # Display Errors\n    display(LD_TRAIN_DF.sort_values('levenshtein_distance', ascending=False).head(100))","metadata":{"execution":{"iopub.status.busy":"2023-08-18T20:20:41.047451Z","iopub.execute_input":"2023-08-18T20:20:41.047884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Value Counts\ndef show_ld_distribution(LD_DF):\n    # Evaluation Metric\n    N = LD_VAL_DF['phrase_true_len'].sum()\n    D = LD_VAL_DF['levenshtein_distance'].sum()\n    nld = (N - D) / N\n    \n    LD_TRAIN_VC = dict([(i, 0) for i in range(LD_DF['levenshtein_distance'].max()+1)])\n    for ld in LD_DF['levenshtein_distance']:\n        LD_TRAIN_VC[ld] += 1\n\n    plt.figure(figsize=(15,8))\n    pd.Series(LD_TRAIN_VC).plot(kind='bar', width=1)\n    plt.title(f'Train Levenstein Distance Distribution | Mean: {LD_DF.levenshtein_distance.mean():.4f}, NLD: {nld:.3f}')\n    plt.xlabel('Levenstein Distance')\n    plt.ylabel('Sample Count')\n    plt.xlim(-0.50, LD_DF.levenshtein_distance.max()+0.50)\n    plt.grid(axis='y')\n    plt.savefig('temp.png')\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    show_ld_distribution(LD_VAL_DF)\nelse:\n    show_ld_distribution(LD_TRAIN_DF)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"# Model Layer Names\nfor l in model.layers:\n    print(l.name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TFLite model for submission\nclass TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n        # Load the feature generation and main models\n        self.preprocess_layer = preprocess_layer\n        self.model = model\n    \n    def encoder(self, x, frames_inp, n_frames):\n        x = self.model.get_layer('embedding')(x, n_frames)\n        x, attention_mask, decoder_wks, decoder_wvs = self.model.get_layer('encoder')(x, frames_inp)\n        # Fill NaN\n        x = tf.where(tf.math.is_nan(x), tf.constant(0.0, x.dtype), x)\n        \n        return x, attention_mask, decoder_wks, decoder_wvs\n        \n    def decoder(self, x, phrase_inp, frames_inp, attention_mask, decoder_wks, decoder_wvs):\n        x = self.model.get_layer('decoder')(x, phrase_inp, frames_inp, attention_mask, decoder_wks, decoder_wvs)\n        x = self.model.get_layer('classifier')(x)\n        # Fill NaN\n        x = tf.where(tf.math.is_nan(x), tf.constant(0.0, x.dtype), x)\n        \n        return x\n    \n    def get_prediction(self, inputs, frames_inp):\n        # Add Batch Dimension\n        frames_inp = tf.expand_dims(frames_inp, axis=0)\n        # Number Of Frames\n        n_frames = len(inputs)\n        n_frames = tf.math.minimum(n_frames, 127*8)\n        # Get Encoding\n        encoding, attention_mask, decoder_wks, decoder_wvs = self.encoder(frames_inp, frames_inp, n_frames)\n        # Make Prediction\n        phrase = tf.fill([1,MAX_PHRASE_LENGTH], PAD_TOKEN)\n        # Predict One Token At A Time\n        stop = False\n        for idx in tf.range(MAX_PRED_LENGTH):\n            # Cast phrase to int8\n            phrase = tf.pad(phrase, [[0,0], [1,0]], constant_values=SOS_TOKEN)\n            phrase = tf.slice(phrase, [0,0], [1,MAX_PHRASE_LENGTH])\n            phrase = tf.cast(phrase, tf.int8)\n            # If EOS token is predicted, stop predicting\n            outputs = tf.cond(\n                stop,\n                lambda: tf.one_hot(tf.cast(phrase, tf.int32), N_UNIQUE_CHARACTERS),\n                lambda: self.decoder(encoding, phrase, frames_inp, attention_mask, decoder_wks, decoder_wvs),\n            )\n            # Catch NaN Outputs\n            outputs = tf.where(tf.math.is_nan(outputs), 0.0, outputs)\n            # Add predicted token to input phrase\n            phrase = tf.cast(phrase, tf.int32)\n            # Replcae PAD token with predicted token up to idx\n            phrase = tf.where(\n                tf.range(MAX_PHRASE_LENGTH) < idx + 1,\n                tf.argmax(outputs, axis=2, output_type=tf.int32),\n                phrase,\n            )\n            # Predicted Token\n            predicted_token = phrase[0,idx]\n            # If EOS (End Of Sentence) token is predicted stop\n            if not stop:\n                stop = predicted_token == EOS_TOKEN\n            \n        # Fill NaN\n        outputs = tf.where(tf.math.is_nan(outputs), tf.constant(0.0, outputs.dtype), outputs)\n            \n        # Squeeze outputs\n        outputs = tf.squeeze(phrase, axis=0)\n        outputs = tf.one_hot(outputs, N_UNIQUE_CHARACTERS)\n            \n        # Return a dictionary with the output tensor\n        return {'outputs': outputs }\n    \n    @tf.function(\n        input_signature=[tf.TensorSpec(shape=[None, N_COLS0], dtype=tf.float32, name='inputs')],\n    )\n    def __call__(self, inputs):\n        # Preprocess Data\n        frames_inp = self.preprocess_layer(inputs)\n        \n        return self.get_prediction(inputs, frames_inp)\n        \n\n# Define TF Lite Model\ntflite_keras_model = TFLiteModel(model)\n\n# Sanity Check\n# demo_sequence_id = 1816796431\ndemo_sequence_id = example_parquet_df.index.unique()[0]\ndemo_raw_data = example_parquet_df.loc[demo_sequence_id, COLUMNS0].values\ndemo_phrase_true = train_sequence_id.loc[demo_sequence_id, 'phrase']\nprint(f'demo_raw_data shape: {demo_raw_data.shape}, dtype: {demo_raw_data.dtype}')\ndemo_output = tflite_keras_model(demo_raw_data)['outputs'].numpy()\nprint(f'demo_output shape: {demo_output.shape}, dtype: {demo_output.dtype}')\nprint(f'demo_outputs phrase decoded: {outputs2phrase(demo_output)}')\nprint(f'phrase true: {demo_phrase_true}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Benchmark Model","metadata":{}},{"cell_type":"code","source":"%%time\ntflite_keras_model(demo_raw_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"demo_sequence_id = example_parquet_df.index.unique()[7]\ndemo_raw_data = example_parquet_df.loc[demo_sequence_id, COLUMNS0].values\ndemo_raw_data.shape\n\n%timeit tflite_keras_model(demo_raw_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess Layer\n%timeit tflite_keras_model.preprocess_layer(demo_raw_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TFLite Model Conversion","metadata":{}},{"cell_type":"code","source":"# Create Model Converter\nkeras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\nkeras_model_converter.experimental_new_converter = True\n# Convert Model\ntflite_model = keras_model_converter.convert()\n# Write Model\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add selected_columns json to only select specific columns from input frames\nwith open('inference_args.json', 'w') as f:\n     json.dump({ 'selected_columns': COLUMNS0.tolist() }, f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Zip Model\n!zip submission.zip /kaggle/working/model.tflite /kaggle/working/inference_args.json","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Verify Correct Inference","metadata":{}},{"cell_type":"code","source":"# # Verify TFLite model can be loaded and used for prediction\n# !pip install tflite-runtime==2.13.0\n# import tflite_runtime.interpreter as tflite\n\n# from tqdm.notebook import tqdm\n\n# import numpy as np\n# import pandas as pd\n\n# import glob\n# import json","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# interpreter = tflite.Interpreter('/kaggle/input/aslfr-inference-python37-tpu-dataset/model.tflite')\n# found_signatures = list(interpreter.get_signature_list().keys())\n# prediction_fn = interpreter.get_signature_runner(\"serving_default\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Dataset Folder\n# DATASET_FOLDER = '/kaggle/input/fsfr-dataset-generation-python37'\n\n# # Unique Parquet Files\n# INFERENCE_FILE_PATHS = pd.Series(\n#         glob.glob(f'{DATASET_FOLDER}/train_landmark_subsets/*')\n#     )\n\n# print(f'Found {len(INFERENCE_FILE_PATHS)} Inference Pickle Files')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Read Character to Ordinal Encoding Mapping\n# with open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n#     CHAR2ORD = json.load(json_file)\n    \n# # Ordinal to Character Mapping\n# ORD2CHAR = {j:i for i,j in CHAR2ORD.items()}\n    \n# # Character to Ordinal Encoding Mapping   \n# display(pd.Series(CHAR2ORD).to_frame('Ordinal Encoding'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Output Predictions to string\n# def outputs2phrase(outputs):\n#     if outputs.ndim == 2:\n#         outputs = np.argmax(outputs, axis=1)\n    \n#     return ''.join([ORD2CHAR.get(s, '') for s in outputs])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Verify Predictions","metadata":{}},{"cell_type":"code","source":"# # Debug flag\n# DEBUG_INFERENCE = False\n\n# # Train\n# train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n\n# SEED = 42\n\n# # Number of parquet chunks to analyse\n# N = len(INFERENCE_FILE_PATHS)\n# # Train DataFrame Indexed By Sequence ID\n# train_squence_id = train.set_index('sequence_id')\n\n# for idx, file_path in enumerate(tqdm(INFERENCE_FILE_PATHS.sample(N, random_state=SEED))):\n#     print(f'file_path: {file_path}')\n#     df = pd.read_parquet(file_path)\n#     for i, (group, group_df) in enumerate(tqdm(df.groupby('sequence_id'))):\n#         inputs = group_df.values\n#         if DEBUG_INFERENCE:\n#             print(f'Attempting to process sequence_id: {group}, inputs shape: {inputs.shape}')\n#         outputs = prediction_fn(inputs=inputs)['outputs']\n#         predicted_phrase = outputs2phrase(outputs).ljust(32)\n#         true_phrase = train_squence_id.loc[group, 'phrase'].ljust(32)\n#         if i < 100:\n#             print(f'outputs shape: {outputs.shape}, pred: {predicted_phrase} true {true_phrase} inputs shape: {inputs.shape}, sequence_id: {group}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}