{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Original link: [https://www.kaggle.com/code/markwijkhuizen/aslfr-transformer-training-inference?scriptVersionId=133874610](https://www.kaggle.com/code/markwijkhuizen/aslfr-transformer-training-inference?scriptVersionId=133874610)\n    \nThanks to @markwijkhuizen for the great code.\n    \nWhat I alter:\n- Add Chinese notes\n- Alter bath size and early stopping\n- Add random noise augmentation.\n\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\nimport tensorflow as tf\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit\nfrom leven import levenshtein\n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport time\nimport json\n\n# TQDM Progress Bar With Pandas Apply Function\ntqdm.pandas()\n\nprint(f'Tensorflow Version {tf.__version__}')\nprint(f'Python Version: {sys.version}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:33:19.281419Z","iopub.execute_input":"2023-06-19T23:33:19.281798Z","iopub.status.idle":"2023-06-19T23:33:27.644234Z","shell.execute_reply.started":"2023-06-19T23:33:19.281768Z","shell.execute_reply":"2023-06-19T23:33:27.643219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 加载字符编码对应关系","metadata":{}},{"cell_type":"code","source":"# Read Character to Ordinal Encoding Mapping\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    CHAR2ORD = json.load(json_file)\n    \n# Ordinal to Character Mapping\nORD2CHAR = {j:i for i,j in CHAR2ORD.items()}\n    \n# Character to Ordinal Encoding Mapping   \ndisplay(pd.Series(CHAR2ORD).to_frame('Ordinal Encoding'))","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:33:27.646246Z","iopub.execute_input":"2023-06-19T23:33:27.646972Z","iopub.status.idle":"2023-06-19T23:33:27.673723Z","shell.execute_reply.started":"2023-06-19T23:33:27.646938Z","shell.execute_reply":"2023-06-19T23:33:27.672778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 全局设置","metadata":{}},{"cell_type":"code","source":"# If Notebook Is Run By Committing or In Interactive Mode For Development\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n# Verbose Setting during training\nVERBOSE = 1 if IS_INTERACTIVE else 2\n# Global Random Seed\nSEED = 42\n# Number of Frames to resize recording to\nN_TARGET_FRAMES = 128\n# Global debug flag, takes subset of train\nDEBUG = False\n# Number of Unique Characters To Predict + Pad Token + SOS Token + EOS Token\nN_UNIQUE_CHARACTERS0 = len(CHAR2ORD)\nN_UNIQUE_CHARACTERS = len(CHAR2ORD) + 1 + 1 + 1\nPAD_TOKEN = len(CHAR2ORD) # Padding\nSOS_TOKEN = len(CHAR2ORD) + 1 # Start Of Sentence\nEOS_TOKEN = len(CHAR2ORD) + 2 # End Of Sentence\n# Whether to use 10% of data for validation\nUSE_VAL = True\n\n\n############################\n# 模型训练设置\n\n# Batch Size\nBATCH_SIZE = 96\n# Number of Epochs to Train for\nN_EPOCHS = 2 if IS_INTERACTIVE else 55\n# Number of Warmup Epochs in Learning Rate Scheduler\nN_WARMUP_EPOCHS = 0\n# Maximum Learning Rate\nLR_MAX = 1e-3\n# Weight Decay Ratio as Ratio of Learning Rate\nWD_RATIO = 0.05\n# Length of Phrase + EOS Token\nMAX_PHRASE_LENGTH = 31 + 1\n# Whether to Train The model\nTRAIN_MODEL = True\n# Whether to Load Pretrained Weights\nLOAD_WEIGHTS = False","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:33:27.675165Z","iopub.execute_input":"2023-06-19T23:33:27.675468Z","iopub.status.idle":"2023-06-19T23:33:27.68317Z","shell.execute_reply.started":"2023-06-19T23:33:27.675444Z","shell.execute_reply":"2023-06-19T23:33:27.682256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 画图设置","metadata":{}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:33:27.685998Z","iopub.execute_input":"2023-06-19T23:33:27.68662Z","iopub.status.idle":"2023-06-19T23:33:27.697603Z","shell.execute_reply.started":"2023-06-19T23:33:27.686589Z","shell.execute_reply":"2023-06-19T23:33:27.696596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 加载训练集","metadata":{}},{"cell_type":"code","source":"# Read Train DataFrame\nif DEBUG:\n    train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv').head(5000)\nelse:\n    train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n    \n# Set Train Indexed By sqeuence_id\ntrain_sequence_id = train.set_index('sequence_id')\n\n# Number Of Train Samples\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}')\n\ndisplay(train.info())\ndisplay(train.head())\n\n# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-fingerspelling/{path}'\ntrain['file_path'] = train['path'].apply(get_file_path)\n\n# Unique Parquet Files\nINFERENCE_FILE_PATHS = pd.Series(\n        glob.glob('/kaggle/input/aslfr-preprocessing-dataset/train_landmark_subsets/*')\n    )\nprint(f'Found {len(INFERENCE_FILE_PATHS)} Inference Pickle Files')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:33:27.698849Z","iopub.execute_input":"2023-06-19T23:33:27.699829Z","iopub.status.idle":"2023-06-19T23:33:27.948148Z","shell.execute_reply.started":"2023-06-19T23:33:27.699798Z","shell.execute_reply":"2023-06-19T23:33:27.947064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 加载处理好的数据和标签","metadata":{}},{"cell_type":"code","source":"# Train/Validation\nif USE_VAL:\n    # TRAIN\n    X_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/X_train.npy')\n    y_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/y_train.npy')[:,:MAX_PHRASE_LENGTH]\n    N_TRAIN_SAMPLES = len(X_train)\n    # VAL\n    X_val = np.load('/kaggle/input/aslfr-preprocessing-dataset/X_val.npy')\n    y_val = np.load('/kaggle/input/aslfr-preprocessing-dataset/y_val.npy')[:,:MAX_PHRASE_LENGTH]\n    N_VAL_SAMPLES = len(X_val)\n    # Shapes\n    print(f'X_train shape: {X_train.shape}, X_val shape: {X_val.shape}')\n# Train On All Data\nelse:\n    # TRAIN\n    X_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/X.npy')\n    y_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/y.npy')[:,:MAX_PHRASE_LENGTH]\n    N_TRAIN_SAMPLES = len(X_train)\n    print(f'X_train shape: {X_train.shape}')\n    \n# Mean/Standard Deviations of data used for normalizing\nMEANS = np.load('/kaggle/input/aslfr-preprocessing-dataset/MEANS.npy').reshape(-1)\nSTDS = np.load('/kaggle/input/aslfr-preprocessing-dataset/STDS.npy').reshape(-1)\n    \n# Example Batch For Debugging\nN_EXAMPLE_BATCH_SAMPLES = 1024\n\nX_batch = {\n    'frames': np.copy(X_train[:N_EXAMPLE_BATCH_SAMPLES]),\n    'phrase': np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES]),\n}\ny_batch = np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES])\n\n# Read First Parquet File\n# example_parquet_df = pd.read_parquet(train['file_path'][0])\nexample_parquet_df = pd.read_parquet(INFERENCE_FILE_PATHS[0])\n\n# Each parquet file contains 1000 recordings\nprint(f'# Unique Recording: {example_parquet_df.index.nunique()}')\n# Display DataFrame layout\ndisplay(example_parquet_df.head())","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:33:27.949865Z","iopub.execute_input":"2023-06-19T23:33:27.950291Z","iopub.status.idle":"2023-06-19T23:34:11.281638Z","shell.execute_reply.started":"2023-06-19T23:33:27.950257Z","shell.execute_reply":"2023-06-19T23:34:11.280704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Landmark 筛选","metadata":{}},{"cell_type":"code","source":"# Get indices in original dataframe\ndef get_idxs(df, words_pos, words_neg=[], ret_names=True, idxs_pos=None):\n    idxs = []\n    names = []\n    for w in words_pos:\n        for col_idx, col in enumerate(example_parquet_df.columns):\n            # Exclude Non Landmark Columns\n            if col in ['frame']:\n                continue\n                \n            col_idx = int(col.split('_')[-1])\n            # Check if column name contains all words\n            if (w in col) and (idxs_pos is None or col_idx in idxs_pos) and all([w not in col for w in words_neg]):\n                idxs.append(col_idx)\n                names.append(col)\n    # Convert to Numpy arrays\n    idxs = np.array(idxs)\n    names = np.array(names)\n    # Returns either both column indices and names\n    if ret_names:\n        return idxs, names\n    # Or only columns indices\n    else:\n        return idxs","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:34:11.283239Z","iopub.execute_input":"2023-06-19T23:34:11.283848Z","iopub.status.idle":"2023-06-19T23:34:11.292445Z","shell.execute_reply.started":"2023-06-19T23:34:11.283814Z","shell.execute_reply":"2023-06-19T23:34:11.291277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lips Landmark Face Ids\nLIPS_LANDMARK_IDXS = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\n\n# Landmark Indices for Left/Right hand without z axis in raw data\nLEFT_HAND_IDXS0, LEFT_HAND_NAMES0 = get_idxs(example_parquet_df, ['left_hand'], ['z'])\nRIGHT_HAND_IDXS0, RIGHT_HAND_NAMES0 = get_idxs(example_parquet_df, ['right_hand'], ['z'])\nLIPS_IDXS0, LIPS_NAMES0 = get_idxs(example_parquet_df, ['face'], ['z'], idxs_pos=LIPS_LANDMARK_IDXS)\nCOLUMNS0 = np.concatenate((LEFT_HAND_NAMES0, RIGHT_HAND_NAMES0, LIPS_NAMES0))\nN_COLS0 = len(COLUMNS0)\n# Only X/Y axes are used\nN_DIMS0 = 2\n\nprint(f'N_COLS0: {N_COLS0}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:34:11.29406Z","iopub.execute_input":"2023-06-19T23:34:11.294495Z","iopub.status.idle":"2023-06-19T23:34:11.311903Z","shell.execute_reply.started":"2023-06-19T23:34:11.294462Z","shell.execute_reply":"2023-06-19T23:34:11.310684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Landmark Indices in subset of dataframe with only COLUMNS selected\nLEFT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, LEFT_HAND_NAMES0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, RIGHT_HAND_NAMES0)).squeeze()\nLIPS_IDXS = np.argwhere(np.isin(COLUMNS0, LIPS_NAMES0)).squeeze()\nHAND_IDXS = np.concatenate((LEFT_HAND_IDXS, RIGHT_HAND_IDXS), axis=0)\nN_COLS = N_COLS0\n# Only X/Y axes are used\nN_DIMS = 2\n\nprint(f'N_COLS: {N_COLS}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:34:11.313441Z","iopub.execute_input":"2023-06-19T23:34:11.314253Z","iopub.status.idle":"2023-06-19T23:34:11.324823Z","shell.execute_reply.started":"2023-06-19T23:34:11.31422Z","shell.execute_reply":"2023-06-19T23:34:11.323827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Indices in processed data by axes with only dominant hand\nHAND_X_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'x' in name]\n    ).squeeze()\nHAND_Y_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'y' in name]\n    ).squeeze()\n# Names in processed data by axes\nHAND_X_NAMES = LEFT_HAND_NAMES0[HAND_X_IDXS]\nHAND_Y_NAMES = LEFT_HAND_NAMES0[HAND_Y_IDXS]","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:34:11.329824Z","iopub.execute_input":"2023-06-19T23:34:11.330401Z","iopub.status.idle":"2023-06-19T23:34:11.336898Z","shell.execute_reply.started":"2023-06-19T23:34:11.330369Z","shell.execute_reply":"2023-06-19T23:34:11.335975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tensorflow 预处理层","metadata":{}},{"cell_type":"code","source":"\"\"\"\n    Tensorflow layer to process data in TFLite\n    Data needs to be processed in the model itself, so we can not use Python\n\"\"\" \nclass PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n        self.normalisation_correction = tf.constant(\n                    # Add 0.50 to x coordinates of left hand (original right hand) and substract 0.50 of right hand (original left hand)\n                     [0.50 if 'x' in name else 0.00 for name in LEFT_HAND_NAMES0],\n                dtype=tf.float32,\n            )\n    \n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,N_COLS0], dtype=tf.float32),),\n    )\n    def call(self, data0, resize=True):\n        # Fill NaN Values With 0\n        data = tf.where(tf.math.is_nan(data0), 0.0, data0)\n        \n        # Hacky\n        data = data[None]\n        \n        # Empty Hand Frame Filtering\n        hands = tf.slice(data, [0,0,0], [-1, -1, 84])\n        hands = tf.abs(hands)\n        mask = tf.reduce_sum(hands, axis=2)\n        mask = tf.not_equal(mask, 0)\n        data = data[mask][None]\n        \n        # Pad Zeros\n        N_FRAMES = len(data[0])\n        if N_FRAMES < N_TARGET_FRAMES:\n            data = tf.concat((\n                data,\n                tf.zeros([1,N_TARGET_FRAMES-N_FRAMES,N_COLS], dtype=tf.float32)\n            ), axis=1)\n        # Downsample\n        data = tf.image.resize(\n            data,\n            [1, N_TARGET_FRAMES],\n            method=tf.image.ResizeMethod.BILINEAR,\n        )\n        \n        # Squeeze Batch Dimension\n        data = tf.squeeze(data, axis=[0])\n        \n        return data\n    \npreprocess_layer = PreprocessLayer()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:34:11.338227Z","iopub.execute_input":"2023-06-19T23:34:11.338876Z","iopub.status.idle":"2023-06-19T23:34:14.562273Z","shell.execute_reply.started":"2023-06-19T23:34:11.338841Z","shell.execute_reply":"2023-06-19T23:34:14.561237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function To Test Preprocessing Layer\ndef test_preprocess_layer():\n    demo_sequence_id = example_parquet_df.index.unique()[15]\n    demo_raw_data = example_parquet_df.loc[demo_sequence_id, COLUMNS0]\n    data = preprocess_layer(demo_raw_data)\n\n    print(f'demo_raw_data shape: {demo_raw_data.shape}')\n    print(f'data shape: {data.shape}')\n    \n    return data\n    \nif IS_INTERACTIVE:\n    data = test_preprocess_layer()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:34:14.563506Z","iopub.execute_input":"2023-06-19T23:34:14.564175Z","iopub.status.idle":"2023-06-19T23:34:14.941352Z","shell.execute_reply.started":"2023-06-19T23:34:14.564141Z","shell.execute_reply":"2023-06-19T23:34:14.940268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 加载训练集","metadata":{}},{"cell_type":"code","source":"# Train Dataset Iterator\ndef get_train_dataset(X, y, batch_size=BATCH_SIZE):\n    sample_idxs = np.arange(len(X))\n    while True:\n        # Get random indices\n        random_sample_idxs = np.random.choice(sample_idxs, batch_size)\n        \n        noise = np.random.uniform(-1,1, size=X[random_sample_idxs].shape) / 20\n        inputs = {\n            'frames': X[random_sample_idxs] # + noise,\n            'phrase': y[random_sample_idxs],\n        }\n        outputs = y[random_sample_idxs]\n        \n        yield inputs, outputs","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:34.045825Z","iopub.execute_input":"2023-06-19T23:43:34.046411Z","iopub.status.idle":"2023-06-19T23:43:34.053058Z","shell.execute_reply.started":"2023-06-19T23:43:34.04638Z","shell.execute_reply":"2023-06-19T23:43:34.051959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Dataset\ntrain_dataset = get_train_dataset(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:34.973535Z","iopub.execute_input":"2023-06-19T23:43:34.974103Z","iopub.status.idle":"2023-06-19T23:43:34.981314Z","shell.execute_reply.started":"2023-06-19T23:43:34.97406Z","shell.execute_reply":"2023-06-19T23:43:34.980327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Steps Per Epoch\nTRAIN_STEPS_PER_EPOCH = math.ceil(N_TRAIN_SAMPLES / BATCH_SIZE)\nprint(f'TRAIN_STEPS_PER_EPOCH: {TRAIN_STEPS_PER_EPOCH}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:39.66942Z","iopub.execute_input":"2023-06-19T23:43:39.669918Z","iopub.status.idle":"2023-06-19T23:43:39.676696Z","shell.execute_reply.started":"2023-06-19T23:43:39.669877Z","shell.execute_reply":"2023-06-19T23:43:39.675731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 加载验证集","metadata":{}},{"cell_type":"code","source":"# Validation Set\ndef get_val_dataset(X, y, batch_size=BATCH_SIZE):\n    offsets = np.arange(0, len(X), batch_size)\n    while True:\n        # Iterate over whole validation set\n        for offset in offsets:\n            inputs = {\n                'frames': X[offset:offset+batch_size],\n                'phrase': y[offset:offset+batch_size],\n            }\n            outputs = y[offset:offset+batch_size]\n\n            yield inputs, outputs","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:41.632119Z","iopub.execute_input":"2023-06-19T23:43:41.632568Z","iopub.status.idle":"2023-06-19T23:43:41.638971Z","shell.execute_reply.started":"2023-06-19T23:43:41.632532Z","shell.execute_reply":"2023-06-19T23:43:41.637785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation Dataset\nif USE_VAL:\n    val_dataset = get_val_dataset(X_val, y_val)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:42.637071Z","iopub.execute_input":"2023-06-19T23:43:42.637916Z","iopub.status.idle":"2023-06-19T23:43:42.642693Z","shell.execute_reply.started":"2023-06-19T23:43:42.63788Z","shell.execute_reply":"2023-06-19T23:43:42.641704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    N_VAL_STEPS_PER_EPOCH = math.ceil(N_VAL_SAMPLES / BATCH_SIZE)\n    print(f'N_VAL_STEPS_PER_EPOCH: {N_VAL_STEPS_PER_EPOCH}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:43.488371Z","iopub.execute_input":"2023-06-19T23:43:43.489086Z","iopub.status.idle":"2023-06-19T23:43:43.494417Z","shell.execute_reply.started":"2023-06-19T23:43:43.489052Z","shell.execute_reply":"2023-06-19T23:43:43.49328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 模型配置","metadata":{}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# final embedding and transformer embedding size\nUNITS_ENCODER = 256\nUNITS_DECODER = 256\n\n# Transformer\nNUM_BLOCKS_ENCODER = 2\nNUM_BLOCKS_DECODER = 2\nMLP_RATIO = 4\n\n# Dropout\nEMBEDDING_DROPOUT = 0.00\nMLP_DROPOUT_RATIO = 0.30\nCLASSIFIER_DROPOUT_RATIO = 0.00\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:44.994175Z","iopub.execute_input":"2023-06-19T23:43:44.995368Z","iopub.status.idle":"2023-06-19T23:43:45.002234Z","shell.execute_reply.started":"2023-06-19T23:43:44.995329Z","shell.execute_reply":"2023-06-19T23:43:45.000764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 模型结构：LandmarkEmbedding","metadata":{}},{"cell_type":"code","source":"# Embeds a landmark using fully connected layers\nclass LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n        \n    def build(self, input_shape):\n        # Embedding for missing landmark in frame, initizlied with zeros\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # Embedding\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.dense(x),\n            )","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:45.919255Z","iopub.execute_input":"2023-06-19T23:43:45.919618Z","iopub.status.idle":"2023-06-19T23:43:45.929694Z","shell.execute_reply.started":"2023-06-19T23:43:45.91959Z","shell.execute_reply":"2023-06-19T23:43:45.928683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 模型结构：Embedding","metadata":{}},{"cell_type":"code","source":"# Creates embedding for each frame\nclass Embedding(tf.keras.Model):\n    def __init__(self):\n        super(Embedding, self).__init__()\n\n    def build(self, input_shape):\n        # Positional embedding for each frame index\n        self.positional_embedding = tf.Variable(\n            initial_value=tf.zeros([N_TARGET_FRAMES, UNITS_ENCODER], dtype=tf.float32),\n            trainable=True,\n            name='embedding_positional_encoder',\n        )\n        # Embedding layer for Landmarks\n        self.dominant_hand_embedding = LandmarkEmbedding(UNITS_ENCODER, 'dominant_hand')\n\n    def call(self, x, training=False):\n        # Normalize\n        x = tf.where(\n                tf.math.equal(x, 0.0),\n                0.0,\n                (x - MEANS) / STDS,\n            )\n        # Dominant Hand\n        x = self.dominant_hand_embedding(x)\n        # Add Positional Encoding\n        x = x + self.positional_embedding\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:46.603647Z","iopub.execute_input":"2023-06-19T23:43:46.604025Z","iopub.status.idle":"2023-06-19T23:43:46.614262Z","shell.execute_reply.started":"2023-06-19T23:43:46.603997Z","shell.execute_reply":"2023-06-19T23:43:46.613303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 模型结构： Transformer","metadata":{}},{"cell_type":"code","source":"# based on: https://stackoverflow.com/questions/67342988/verifying-the-implementation-of-multihead-attention-in-transformer\n# replaced softmax with softmax layer to support masked softmax\ndef scaled_dot_product(q,k,v, softmax, attention_mask):\n    #calculates Q . K(transpose)\n    qkt = tf.matmul(q,k,transpose_b=True)\n    #caculates scaling factor\n    dk = tf.math.sqrt(tf.cast(q.shape[-1],dtype=tf.float32))\n    scaled_qkt = qkt/dk\n    softmax = softmax(scaled_qkt, mask=attention_mask)\n    z = tf.matmul(softmax,v)\n    #shape: (m,Tx,depth), same shape as q,k,v\n    return z\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self,d_model,num_of_heads):\n        super(MultiHeadAttention,self).__init__()\n        self.d_model = d_model\n        self.num_of_heads = num_of_heads\n        self.depth = d_model//num_of_heads\n        self.wq = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wk = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wv = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wo = tf.keras.layers.Dense(d_model)\n        self.softmax = tf.keras.layers.Softmax()\n        \n    def call(self, q, k, v, attention_mask=None):\n        \n        multi_attn = []\n        for i in range(self.num_of_heads):\n            Q = self.wq[i](q)\n            K = self.wk[i](k)\n            V = self.wv[i](v)\n            multi_attn.append(scaled_dot_product(Q,K,V, self.softmax, attention_mask))\n            \n        multi_head = tf.concat(multi_attn, axis=-1)\n        multi_head_attention = self.wo(multi_head)\n        return multi_head_attention","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:47.487602Z","iopub.execute_input":"2023-06-19T23:43:47.488309Z","iopub.status.idle":"2023-06-19T23:43:47.502002Z","shell.execute_reply.started":"2023-06-19T23:43:47.488274Z","shell.execute_reply":"2023-06-19T23:43:47.500673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 模型结构：Encoder\n\n[source](https://keras.io/examples/nlp/neural_machine_translation_with_transformer/)","metadata":{}},{"cell_type":"code","source":"# Encoder based on multiple transformer blocks\nclass Encoder(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Encoder, self).__init__(name='encoder')\n        self.num_blocks = num_blocks\n    \n    def build(self, input_shape):\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS_ENCODER, 8))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS_ENCODER * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS_ENCODER, kernel_initializer=INIT_HE_UNIFORM),\n            ]))\n        \n    def call(self, x, x_inp):\n        # Attention mask to ignore missing frames\n        attention_mask = tf.where(tf.math.reduce_sum(x_inp, axis=[2]) == 0.0, 0.0, 1.0)\n        attention_mask = tf.expand_dims(attention_mask, axis=1)\n        attention_mask = tf.repeat(attention_mask, repeats=N_TARGET_FRAMES, axis=1)\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x = ln_1(x + mha(x, x, x, attention_mask=attention_mask))\n            x = ln_2(x + mlp(x))\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:48.638526Z","iopub.execute_input":"2023-06-19T23:43:48.638924Z","iopub.status.idle":"2023-06-19T23:43:48.651771Z","shell.execute_reply.started":"2023-06-19T23:43:48.638889Z","shell.execute_reply":"2023-06-19T23:43:48.650696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 模型结构：Decoder","metadata":{}},{"cell_type":"code","source":"# Decoder based on multiple transformer blocks\nclass Decoder(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Decoder, self).__init__(name='decoder')\n        self.num_blocks = num_blocks\n    \n    def build(self, input_shape):\n        # Positional Embedding, initialized with zeros\n        self.positional_embedding = tf.Variable(\n            initial_value=tf.zeros([N_TARGET_FRAMES, UNITS_DECODER], dtype=tf.float32),\n            trainable=True,\n            name='embedding_positional_encoder',\n        )\n        # Character Embedding\n        self.char_emb = tf.keras.layers.Embedding(N_UNIQUE_CHARACTERS, UNITS_DECODER, embeddings_initializer=INIT_ZEROS)\n        # Positional Encoder MHA\n        self.pos_emb_mha = MultiHeadAttention(UNITS_DECODER, 8)\n        self.pos_emb_ln = tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS)\n        # First Layer Normalisation\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS_DECODER, 8))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS_DECODER * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS_DECODER, kernel_initializer=INIT_HE_UNIFORM),\n            ]))\n            \n    def get_causal_attention_mask(self, B):\n        i = tf.range(N_TARGET_FRAMES)[:, tf.newaxis]\n        j = tf.range(N_TARGET_FRAMES)\n        mask = tf.cast(i >= j, dtype=tf.int32)\n        mask = tf.reshape(mask, (1, N_TARGET_FRAMES, N_TARGET_FRAMES))\n        mult = tf.concat(\n            [tf.expand_dims(B, -1), tf.constant([1, 1], dtype=tf.int32)],\n            axis=0,\n        )\n        mask = tf.tile(mask, mult)\n        mask = tf.cast(mask, tf.float32)\n        return mask\n        \n    def call(self, encoder_outputs, phrase):\n        # Batch Size\n        B = tf.shape(encoder_outputs)[0]\n        # Cast to INT32\n        phrase = tf.cast(phrase, tf.int32)\n        # Prepend SOS Token\n        phrase = tf.pad(phrase, [[0,0], [1,0]], constant_values=SOS_TOKEN, name='prepend_sos_token')\n        # Pad With PAD Token\n        phrase = tf.pad(phrase, [[0,0], [0,N_TARGET_FRAMES-MAX_PHRASE_LENGTH-1]], constant_values=PAD_TOKEN, name='append_pad_token')\n        # Causal Mask\n        causal_mask = self.get_causal_attention_mask(B)\n        # Positional Embedding\n        x = self.positional_embedding + self.char_emb(phrase)\n        # Causal Attention\n        x = self.pos_emb_ln(x + self.pos_emb_mha(x, x, x, attention_mask=causal_mask))\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x = ln_1(x + mha(x, encoder_outputs, encoder_outputs, attention_mask=causal_mask))\n            x = ln_2(x + mlp(x))\n        # Slice 31 Characters\n        x = tf.slice(x, [0, 0, 0], [-1, MAX_PHRASE_LENGTH, -1])\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:49.616692Z","iopub.execute_input":"2023-06-19T23:43:49.617243Z","iopub.status.idle":"2023-06-19T23:43:49.636381Z","shell.execute_reply.started":"2023-06-19T23:43:49.617209Z","shell.execute_reply":"2023-06-19T23:43:49.635311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Causal Attention to make decoder not attent to future characters which it needs to predict\ndef get_causal_attention_mask(B):\n    i = tf.range(N_TARGET_FRAMES)[:, tf.newaxis]\n    j = tf.range(N_TARGET_FRAMES)\n    mask = tf.cast(i >= j, dtype=tf.int32)\n    mask = tf.reshape(mask, (1, N_TARGET_FRAMES, N_TARGET_FRAMES))\n    mult = tf.concat(\n        [tf.expand_dims(B, -1), tf.constant([1, 1], dtype=tf.int32)],\n        axis=0,\n    )\n    mask = tf.tile(mask, mult)\n    mask = tf.cast(mask, tf.float32)\n    return mask\n\nget_causal_attention_mask(1)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:50.141147Z","iopub.execute_input":"2023-06-19T23:43:50.142051Z","iopub.status.idle":"2023-06-19T23:43:50.580662Z","shell.execute_reply.started":"2023-06-19T23:43:50.142008Z","shell.execute_reply":"2023-06-19T23:43:50.579763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 评价指标","metadata":{}},{"cell_type":"code","source":"# TopK accuracy for multi dimensional output\nclass TopKAccuracy(tf.keras.metrics.Metric):\n    def __init__(self, k, **kwargs):\n        super(TopKAccuracy, self).__init__(name=f'top{k}acc', **kwargs)\n        self.top_k_acc = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=k)\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        y_true = tf.reshape(y_true, [-1])\n        y_pred = tf.reshape(y_pred, [-1, N_UNIQUE_CHARACTERS])\n        character_idxs = tf.where(y_true < N_UNIQUE_CHARACTERS0)\n        y_true = tf.gather(y_true, character_idxs, axis=0)\n        y_pred = tf.gather(y_pred, character_idxs, axis=0)\n        self.top_k_acc.update_state(y_true, y_pred)\n\n    def result(self):\n        return self.top_k_acc.result()\n    \n    def reset_state(self):\n        self.top_k_acc.reset_state()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:51.023487Z","iopub.execute_input":"2023-06-19T23:43:51.024246Z","iopub.status.idle":"2023-06-19T23:43:51.033091Z","shell.execute_reply.started":"2023-06-19T23:43:51.0242Z","shell.execute_reply":"2023-06-19T23:43:51.03188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss Weights","metadata":{}},{"cell_type":"code","source":"# Create Initial Loss Weights All Set To 1\nloss_weights = np.ones(N_UNIQUE_CHARACTERS, dtype=np.float32)\n# Set Loss Weight Of Pad Token To 0\nloss_weights[PAD_TOKEN] = 0","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:51.900762Z","iopub.execute_input":"2023-06-19T23:43:51.901126Z","iopub.status.idle":"2023-06-19T23:43:51.906434Z","shell.execute_reply.started":"2023-06-19T23:43:51.901099Z","shell.execute_reply":"2023-06-19T23:43:51.905176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sparse Categorical Crossentropy With Label Smoothing¶","metadata":{}},{"cell_type":"code","source":"# source:: https://stackoverflow.com/questions/60689185/label-smoothing-for-sparse-categorical-crossentropy\ndef scce_with_ls(y_true, y_pred):\n    # One Hot Encode Sparsely Encoded Target Sign\n    y_true = tf.cast(y_true, tf.int32)\n    y_true = tf.one_hot(y_true, N_UNIQUE_CHARACTERS, axis=2)\n    # Categorical Crossentropy with native label smoothing support\n    return tf.keras.losses.categorical_crossentropy(y_true, y_pred, label_smoothing=0.25)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:53.603378Z","iopub.execute_input":"2023-06-19T23:43:53.60376Z","iopub.status.idle":"2023-06-19T23:43:53.610104Z","shell.execute_reply.started":"2023-06-19T23:43:53.603721Z","shell.execute_reply":"2023-06-19T23:43:53.608766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 模型结构","metadata":{}},{"cell_type":"code","source":"def get_model():\n    # Inputs\n    frames_inp = tf.keras.layers.Input([N_TARGET_FRAMES, N_COLS], dtype=tf.float32, name='frames')\n    phrase_inp = tf.keras.layers.Input([MAX_PHRASE_LENGTH], dtype=tf.int32, name='phrase')\n    # Frames\n    x = frames_inp\n\n    # Embedding\n    x = Embedding()(x, frames_inp)\n    \n    # Encoder Transformer Blocks\n    x = Encoder(NUM_BLOCKS_ENCODER)(x, frames_inp)\n    \n    # Decoder\n    x = Decoder(NUM_BLOCKS_DECODER)(x, phrase_inp)\n    \n    # Classifier\n    x = tf.keras.Sequential([\n        # Dropout\n        tf.keras.layers.Dropout(CLASSIFIER_DROPOUT_RATIO),\n        # Output Neurons\n        tf.keras.layers.Dense(N_UNIQUE_CHARACTERS, activation=tf.keras.activations.softmax, kernel_initializer=INIT_HE_UNIFORM),\n    ], name='classifier')(x)\n    \n    outputs = x\n    \n    # Create Tensorflow Model\n    model = tf.keras.models.Model(inputs=[frames_inp, phrase_inp], outputs=outputs)\n    \n    # Simple Categorical Crossentropy Loss\n#     loss = tf.keras.losses.SparseCategoricalCrossentropy()\n    # Categorical Crossentropy Loss With Label Smoothing\n    loss = scce_with_ls\n    \n    # Adam Optimizer with weight decay\n    optimizer = tf.keras.optimizers.Adam(clipnorm=5.0)\n    \n    # TopK Metrics\n    metrics = [\n        TopKAccuracy(1),\n        TopKAccuracy(5),\n    ]\n    \n    model.compile(\n        loss=loss,\n        optimizer=optimizer,\n        metrics=metrics,\n        loss_weights=loss_weights,\n    )\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:54.761226Z","iopub.execute_input":"2023-06-19T23:43:54.762249Z","iopub.status.idle":"2023-06-19T23:43:54.773512Z","shell.execute_reply.started":"2023-06-19T23:43:54.7622Z","shell.execute_reply":"2023-06-19T23:43:54.772525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Input data\nfor k, v in X_batch.items():\n    print(f'{k}: {v.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:55.381111Z","iopub.execute_input":"2023-06-19T23:43:55.381992Z","iopub.status.idle":"2023-06-19T23:43:55.387863Z","shell.execute_reply.started":"2023-06-19T23:43:55.38195Z","shell.execute_reply":"2023-06-19T23:43:55.386551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = get_model()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:56.194449Z","iopub.execute_input":"2023-06-19T23:43:56.194832Z","iopub.status.idle":"2023-06-19T23:43:59.622826Z","shell.execute_reply.started":"2023-06-19T23:43:56.194802Z","shell.execute_reply":"2023-06-19T23:43:59.621884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:43:59.624674Z","iopub.execute_input":"2023-06-19T23:43:59.62504Z","iopub.status.idle":"2023-06-19T23:43:59.763455Z","shell.execute_reply.started":"2023-06-19T23:43:59.625007Z","shell.execute_reply":"2023-06-19T23:43:59.76273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Model Architecture\ntf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True, show_layer_activations=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:44:01.637972Z","iopub.execute_input":"2023-06-19T23:44:01.638422Z","iopub.status.idle":"2023-06-19T23:44:01.890631Z","shell.execute_reply.started":"2023-06-19T23:44:01.638386Z","shell.execute_reply":"2023-06-19T23:44:01.889728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 验证数据集","metadata":{}},{"cell_type":"code","source":"# Dummy Prediction\ny_pred = model.predict(\n    val_dataset if USE_VAL else train_dataset,\n    steps=N_VAL_STEPS_PER_EPOCH if USE_VAL else TRAIN_STEPS_PER_EPOCH,\n    verbose=VERBOSE,\n)\n\nprint(f'# NaN Values In Predictions: {np.isnan(y_pred).sum()}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:44:03.054519Z","iopub.execute_input":"2023-06-19T23:44:03.054904Z","iopub.status.idle":"2023-06-19T23:44:12.297354Z","shell.execute_reply.started":"2023-06-19T23:44:03.054869Z","shell.execute_reply":"2023-06-19T23:44:12.296315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 设置学习率","metadata":{}},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n    \n    if current_step < num_warmup_steps:\n        if WARMUP_METHOD == 'log':\n            return lr_max * 0.10 ** (num_warmup_steps - current_step)\n        else:\n            return lr_max * 2 ** -(num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:44:14.698943Z","iopub.execute_input":"2023-06-19T23:44:14.699317Z","iopub.status.idle":"2023-06-19T23:44:14.706227Z","shell.execute_reply.started":"2023-06-19T23:44:14.699289Z","shell.execute_reply":"2023-06-19T23:44:14.705275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule, epochs):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(1, epochs + 1)\n    x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n    plt.xlim([1, epochs])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n    # Increase y-limit for better readability\n    plt.ylim([0, max(lr_schedule) * 1.1])\n    \n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n            if x < len(lr_schedule) - 1:\n                if lr_schedule[x - 1] < val:\n                    ha = 'right'\n                else:\n                    ha = 'left'\n            elif x == 0:\n                ha = 'right'\n            else:\n                ha = 'left'\n            plt.plot(x + 1, val, 'o', color='black');\n            offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n            plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=0)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:44:15.431647Z","iopub.execute_input":"2023-06-19T23:44:15.43236Z","iopub.status.idle":"2023-06-19T23:44:15.784683Z","shell.execute_reply.started":"2023-06-19T23:44:15.432324Z","shell.execute_reply":"2023-06-19T23:44:15.78363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 设置WeightDecay","metadata":{}},{"cell_type":"code","source":"# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:44:16.599227Z","iopub.execute_input":"2023-06-19T23:44:16.599963Z","iopub.status.idle":"2023-06-19T23:44:16.606308Z","shell.execute_reply.started":"2023-06-19T23:44:16.599919Z","shell.execute_reply":"2023-06-19T23:44:16.60534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate Initialized Model On Validation Data\ny_pred = model.evaluate(\n    val_dataset if USE_VAL else train_dataset,\n    steps=N_VAL_STEPS_PER_EPOCH if USE_VAL else TRAIN_STEPS_PER_EPOCH,\n    verbose=VERBOSE,\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:44:17.986377Z","iopub.execute_input":"2023-06-19T23:44:17.986766Z","iopub.status.idle":"2023-06-19T23:44:27.494638Z","shell.execute_reply.started":"2023-06-19T23:44:17.986727Z","shell.execute_reply":"2023-06-19T23:44:27.493704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# baseline accuracy when only pad token is predicted\nif USE_VAL:\n    baseline_accuracy = np.mean(y_val == PAD_TOKEN)\nelse:\n    baseline_accuracy = np.mean(y_train == PAD_TOKEN)\nprint(f'Baseline Accuracy: {baseline_accuracy:.4f}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:45:03.743386Z","iopub.execute_input":"2023-06-19T23:45:03.743793Z","iopub.status.idle":"2023-06-19T23:45:03.750922Z","shell.execute_reply.started":"2023-06-19T23:45:03.743761Z","shell.execute_reply":"2023-06-19T23:45:03.749896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 模型训练","metadata":{}},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:45:05.477916Z","iopub.execute_input":"2023-06-19T23:45:05.478287Z","iopub.status.idle":"2023-06-19T23:45:45.920679Z","shell.execute_reply.started":"2023-06-19T23:45:05.478259Z","shell.execute_reply":"2023-06-19T23:45:45.919709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ealystop_callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)\n\nif TRAIN_MODEL:\n    # Clear all models in GPU\n    tf.keras.backend.clear_session()\n\n    # Get new fresh model\n    model = get_model()\n\n    # Sanity Check\n    model.summary()\n\n    # Actual Training\n    history = model.fit(\n            x=train_dataset,\n            steps_per_epoch=TRAIN_STEPS_PER_EPOCH,\n            epochs=N_EPOCHS,\n            # Only used for validation data since training data is a generator\n            validation_data=val_dataset if USE_VAL else None,\n            validation_steps=N_VAL_STEPS_PER_EPOCH if USE_VAL else None,\n            callbacks=[\n                lr_callback,\n                WeightDecayCallback(),\n                ealystop_callback\n            ],\n            verbose = VERBOSE,\n        )","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:45:45.924279Z","iopub.execute_input":"2023-06-19T23:45:45.924582Z","iopub.status.idle":"2023-06-19T23:50:54.210685Z","shell.execute_reply.started":"2023-06-19T23:45:45.924557Z","shell.execute_reply":"2023-06-19T23:50:54.209628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load Weights\nif LOAD_WEIGHTS:\n    model.load_weights('/kaggle/input/aslfr-training-python37/model.h5')\n    print(f'Successfully Loaded Pretrained Weights')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.574313Z","iopub.status.idle":"2023-06-19T23:36:46.574811Z","shell.execute_reply.started":"2023-06-19T23:36:46.574547Z","shell.execute_reply":"2023-06-19T23:36:46.57457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save Model Weights\nmodel.save_weights('model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.576554Z","iopub.status.idle":"2023-06-19T23:36:46.577035Z","shell.execute_reply.started":"2023-06-19T23:36:46.576792Z","shell.execute_reply":"2023-06-19T23:36:46.576813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify Model is Loaded Correctly\nmodel.evaluate(\n    val_dataset if USE_VAL else train_dataset,\n    steps=N_VAL_STEPS_PER_EPOCH if USE_VAL else TRAIN_STEPS_PER_EPOCH,\n    batch_size=BATCH_SIZE,\n    verbose=VERBOSE,\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.578698Z","iopub.status.idle":"2023-06-19T23:36:46.579487Z","shell.execute_reply.started":"2023-06-19T23:36:46.579224Z","shell.execute_reply":"2023-06-19T23:36:46.579247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 验证Levenshtein Distance","metadata":{}},{"cell_type":"code","source":"# Output Predictions to string\ndef outputs2phrase(outputs):\n    if outputs.ndim == 2:\n        outputs = np.argmax(outputs, axis=1)\n    \n    return ''.join([ORD2CHAR.get(s, '') for s in outputs])","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.580704Z","iopub.status.idle":"2023-06-19T23:36:46.581565Z","shell.execute_reply.started":"2023-06-19T23:36:46.581312Z","shell.execute_reply":"2023-06-19T23:36:46.581334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function()\ndef predict_phrase(frames):\n    # Add Batch Dimension\n    frames = tf.expand_dims(frames, axis=0)\n    # Start Phrase\n    phrase = tf.fill([1,MAX_PHRASE_LENGTH], PAD_TOKEN)\n\n    for idx in tf.range(MAX_PHRASE_LENGTH):\n        # Cast phrase to int8\n        phrase = tf.cast(phrase, tf.int8)\n        # Predict Next Token\n        outputs = model({\n            'frames': frames,\n            'phrase': phrase,\n        })\n\n        # Add predicted token to input phrase\n        phrase = tf.cast(phrase, tf.int32)\n        phrase = tf.where(\n            tf.range(MAX_PHRASE_LENGTH) < idx + 1,\n            tf.argmax(outputs, axis=2, output_type=tf.int32),\n            phrase,\n        )\n\n    # Squeeze outputs\n    outputs = tf.squeeze(phrase, axis=0)\n    outputs = tf.one_hot(outputs, N_UNIQUE_CHARACTERS)\n\n    # Return a dictionary with the output tensor\n    return outputs\n\n    # Return a dictionary with the output tensor\n    return outputs","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.583023Z","iopub.status.idle":"2023-06-19T23:36:46.583558Z","shell.execute_reply.started":"2023-06-19T23:36:46.583306Z","shell.execute_reply":"2023-06-19T23:36:46.583327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute Levenstein Distances\ndef get_ld_train():\n    N = 100 if IS_INTERACTIVE else 1000\n    LD_TRAIN = []\n    for idx, (frames, phrase_true) in enumerate(zip(tqdm(X_train, total=N), y_train)):\n        # Predict Phrase and Convert to String\n        phrase_pred = predict_phrase(frames).numpy()\n        phrase_pred = outputs2phrase(phrase_pred)\n        # True Phrase Ordinal to String\n        phrase_true = outputs2phrase(phrase_true)\n        # Add Levenstein Distance\n        LD_TRAIN.append({\n            'phrase_true': phrase_true,\n            'phrase_pred': phrase_pred,\n            'levenshtein_distance': levenshtein(phrase_pred, phrase_true),\n        })\n        # Take subset in interactive mode\n        if idx == N:\n            break\n            \n    # Convert to DataFrame\n    LD_TRAIN_DF = pd.DataFrame(LD_TRAIN)\n    \n    return LD_TRAIN_DF","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.585333Z","iopub.status.idle":"2023-06-19T23:36:46.585819Z","shell.execute_reply.started":"2023-06-19T23:36:46.585564Z","shell.execute_reply":"2023-06-19T23:36:46.585585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LD_TRAIN_DF = get_ld_train()\n\n# Display Errors\ndisplay(LD_TRAIN_DF.head(30))","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.587359Z","iopub.status.idle":"2023-06-19T23:36:46.587848Z","shell.execute_reply.started":"2023-06-19T23:36:46.587596Z","shell.execute_reply":"2023-06-19T23:36:46.587618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Value Counts\nLD_TRAIN_VC = dict([(i, 0) for i in range(LD_TRAIN_DF['levenshtein_distance'].max()+1)])\nfor ld in LD_TRAIN_DF['levenshtein_distance']:\n    LD_TRAIN_VC[ld] += 1\n\nplt.figure(figsize=(15,8))\npd.Series(LD_TRAIN_VC).plot(kind='bar', width=1)\nplt.title(f'Train Levenstein Distance Distribution | Mean: {LD_TRAIN_DF.levenshtein_distance.mean():.4f}')\nplt.xlabel('Levenstein Distance')\nplt.ylabel('Sample Count')\nplt.xlim(-0.50, LD_TRAIN_DF.levenshtein_distance.max()+0.50)\nplt.grid(axis='y')\nplt.savefig('temp.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.58943Z","iopub.status.idle":"2023-06-19T23:36:46.589903Z","shell.execute_reply.started":"2023-06-19T23:36:46.589651Z","shell.execute_reply":"2023-06-19T23:36:46.589672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute Levenstein Distances\ndef get_ld_val():\n    N = 100 if IS_INTERACTIVE else 1000\n    LD_VAL = []\n    for idx, (frames, phrase_true) in enumerate(zip(tqdm(X_val, total=N), y_val)):\n        # Predict Phrase and Convert to String\n        phrase_pred = predict_phrase(frames).numpy()\n        phrase_pred = outputs2phrase(phrase_pred)\n        # True Phrase Ordinal to String\n        phrase_true = outputs2phrase(phrase_true)\n        # Add Levenstein Distance\n        LD_VAL.append({\n            'phrase_true': phrase_true,\n            'phrase_pred': phrase_pred,\n            'levenshtein_distance': levenshtein(phrase_pred, phrase_true),\n        })\n        # Take subset in interactive mode\n        if idx == N:\n            break\n            \n    # Convert to DataFrame\n    LD_VAL_DF = pd.DataFrame(LD_VAL)\n    \n    return LD_VAL_DF","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.592302Z","iopub.status.idle":"2023-06-19T23:36:46.593405Z","shell.execute_reply.started":"2023-06-19T23:36:46.593158Z","shell.execute_reply":"2023-06-19T23:36:46.59318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    LD_VAL_DF = get_ld_val()\n\n    # Display Errors\n    display(LD_VAL_DF.head(30))","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.594614Z","iopub.status.idle":"2023-06-19T23:36:46.595572Z","shell.execute_reply.started":"2023-06-19T23:36:46.595319Z","shell.execute_reply":"2023-06-19T23:36:46.595347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Value Counts\nif USE_VAL:\n    LD_VAL_VC = dict([(i, 0) for i in range(LD_VAL_DF['levenshtein_distance'].max()+1)])\n    for ld in LD_VAL_DF['levenshtein_distance']:\n        LD_VAL_VC[ld] += 1\n\n    plt.figure(figsize=(15,8))\n    pd.Series(LD_VAL_VC).plot(kind='bar', width=1)\n    plt.title(f'Validation Levenstein Distance Distribution | Mean: {LD_VAL_DF.levenshtein_distance.mean():.4f}')\n    plt.xlabel('Levenstein Distance')\n    plt.ylabel('Sample Count')\n    plt.xlim(0-0.50, LD_VAL_DF.levenshtein_distance.max()+0.50)\n    plt.grid(axis='y')\n    plt.savefig('temp.png')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.596962Z","iopub.status.idle":"2023-06-19T23:36:46.597425Z","shell.execute_reply.started":"2023-06-19T23:36:46.597184Z","shell.execute_reply":"2023-06-19T23:36:46.597206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 打印训练日志","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax, ylim=None, yscale=None, yticks=None):\n    # Only plot when training\n    if not TRAIN_MODEL:\n        return\n    \n    plt.figure(figsize=(20, 10))\n    \n    values = history.history[metric]\n    N_EPOCHS = len(values)\n    val = 'val' in ''.join(history.history.keys())\n    # Epoch Ticks\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n\n    x_ticks = np.arange(1, N_EPOCHS+1)\n\n    # Validation\n    if val:\n        val_values = history.history[f'val_{metric}']\n        val_argmin = f_best(val_values)\n        plt.plot(x_ticks, val_values, label=f'val')\n\n    # summarize history for accuracy\n    plt.plot(x_ticks, values, label=f'train')\n    argmin = f_best(values)\n    plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label=f'train_best')\n    if val:\n        plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label=f'val_best')\n\n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n\n    if ylim:\n        plt.ylim(ylim)\n\n    if yscale is not None:\n        plt.yscale(yscale)\n        \n    if yticks is not None:\n        plt.yticks(yticks, fontsize=16)\n\n    plt.xlabel('epoch', fontsize=20, labelpad=10)        \n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    \n    plt.legend(prop={'size': 10})\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.599536Z","iopub.status.idle":"2023-06-19T23:36:46.600638Z","shell.execute_reply.started":"2023-06-19T23:36:46.600386Z","shell.execute_reply":"2023-06-19T23:36:46.600409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.601869Z","iopub.status.idle":"2023-06-19T23:36:46.602658Z","shell.execute_reply.started":"2023-06-19T23:36:46.602417Z","shell.execute_reply":"2023-06-19T23:36:46.602439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('top1acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.604138Z","iopub.status.idle":"2023-06-19T23:36:46.605305Z","shell.execute_reply.started":"2023-06-19T23:36:46.605024Z","shell.execute_reply":"2023-06-19T23:36:46.60505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('top5acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.606624Z","iopub.status.idle":"2023-06-19T23:36:46.607664Z","shell.execute_reply.started":"2023-06-19T23:36:46.607421Z","shell.execute_reply":"2023-06-19T23:36:46.607443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 导出模型为Lite","metadata":{}},{"cell_type":"code","source":"# Model Layer Names\nfor l in model.layers:\n    print(l.name)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.608885Z","iopub.status.idle":"2023-06-19T23:36:46.60992Z","shell.execute_reply.started":"2023-06-19T23:36:46.609659Z","shell.execute_reply":"2023-06-19T23:36:46.609682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TFLite model for submission\nclass TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n\n        # Load the feature generation and main models\n        self.preprocess_layer = preprocess_layer\n        self.model = model\n    \n    @tf.function(jit_compile=True)\n    def encoder(self, x, frames_inp):\n        x = self.model.get_layer('embedding')(x, frames_inp)\n        x = self.model.get_layer('encoder')(x, frames_inp)\n        \n        return x\n        \n    @tf.function(jit_compile=True)\n    def decoder(self, x, phrase_inp):\n        x = self.model.get_layer('decoder')(x, phrase_inp)\n        x = self.model.get_layer('classifier')(x)\n        \n        return x\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, N_COLS0], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs):\n        # Number Of Input Frames\n        N_INPUT_FRAMES = tf.shape(inputs)[0]\n        # Preprocess Data\n        frames_inp = self.preprocess_layer(inputs)        \n        # Add Batch Dimension\n        frames_inp = tf.expand_dims(frames_inp, axis=0)\n        # Get Encoding\n        encoding = self.encoder(frames_inp, frames_inp)\n        # Make Prediction\n        phrase = tf.fill([1,MAX_PHRASE_LENGTH], PAD_TOKEN)\n        # Predict One Token At A Time\n        stop = False\n        for idx in tf.range(MAX_PHRASE_LENGTH):\n            # Cast phrase to int8\n            phrase = tf.cast(phrase, tf.int8)\n            # If EOS token is predicted, stop predicting\n            outputs = tf.cond(\n                stop,\n                lambda: tf.one_hot(tf.cast(phrase, tf.int32), N_UNIQUE_CHARACTERS),\n                lambda: self.decoder(encoding, phrase)\n            )\n            # Add predicted token to input phrase\n            phrase = tf.cast(phrase, tf.int32)\n            # Replcae PAD token with predicted token up to idx\n            phrase = tf.where(\n                tf.range(MAX_PHRASE_LENGTH) < idx + 1,\n                tf.argmax(outputs, axis=2, output_type=tf.int32),\n                phrase,\n            )\n            # Predicted Token\n            predicted_token = phrase[0,idx]\n            # If EOS (End Of Sentence) token is predicted stop\n            if not stop:\n                stop = predicted_token == EOS_TOKEN\n            \n        # Squeeze outputs\n        outputs = tf.squeeze(phrase, axis=0)\n        outputs = tf.one_hot(outputs, N_UNIQUE_CHARACTERS)\n            \n        # Return a dictionary with the output tensor\n        return {'outputs': outputs }\n\n# Define TF Lite Model\ntflite_keras_model = TFLiteModel(model)\n\n# Sanity Check\n# demo_sequence_id = 1816796431\ndemo_sequence_id = example_parquet_df.index.unique()[0]\ndemo_raw_data = example_parquet_df.loc[demo_sequence_id, COLUMNS0].values\ndemo_phrase_true = train_sequence_id.loc[demo_sequence_id, 'phrase']\nprint(f'demo_raw_data shape: {demo_raw_data.shape}, dtype: {demo_raw_data.dtype}')\ndemo_output = tflite_keras_model(demo_raw_data)['outputs'].numpy()\nprint(f'demo_output shape: {demo_output.shape}, dtype: {demo_output.dtype}')\nprint(f'demo_outputs phrase decoded: {outputs2phrase(demo_output)}')\nprint(f'phrase true: {demo_phrase_true}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.611189Z","iopub.status.idle":"2023-06-19T23:36:46.611839Z","shell.execute_reply.started":"2023-06-19T23:36:46.611578Z","shell.execute_reply":"2023-06-19T23:36:46.6116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Model Converter\nkeras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\n# Convert Model\ntflite_model = keras_model_converter.convert()\n# Write Model\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.613308Z","iopub.status.idle":"2023-06-19T23:36:46.614233Z","shell.execute_reply.started":"2023-06-19T23:36:46.613988Z","shell.execute_reply":"2023-06-19T23:36:46.61401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add selected_columns json to only select specific columns from input frames\nwith open('inference_args.json', 'w') as f:\n     json.dump({ 'selected_columns': COLUMNS0.tolist() }, f)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.615582Z","iopub.status.idle":"2023-06-19T23:36:46.616066Z","shell.execute_reply.started":"2023-06-19T23:36:46.615817Z","shell.execute_reply":"2023-06-19T23:36:46.615839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Zip Model\n!zip submission.zip /kaggle/working/model.tflite /kaggle/working/inference_args.json","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.617659Z","iopub.status.idle":"2023-06-19T23:36:46.61814Z","shell.execute_reply.started":"2023-06-19T23:36:46.617909Z","shell.execute_reply":"2023-06-19T23:36:46.61793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -f inference_args.json model.h5 model.png model.tflite temp.png","metadata":{"execution":{"iopub.status.busy":"2023-06-19T23:36:46.619501Z","iopub.status.idle":"2023-06-19T23:36:46.62029Z","shell.execute_reply.started":"2023-06-19T23:36:46.620054Z","shell.execute_reply":"2023-06-19T23:36:46.620077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}