{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Package","metadata":{"id":"NCTUih8OTTK5"}},{"cell_type":"code","source":"#addface_4pose\n#holdnosmallhand\n#256frame\n#gelu\n#no_valid\n#addmotionfeature\n#addcon3dtf\n#jicheng supplement model\n!pip install -q tensorflow-addons==0.20.0\n!pip install -q git+https://github.com/hoyso48/tf-utils@main\nimport gc\nimport json\nimport math\nimport pickle\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport time\nfrom tf_utils.learners import FGM, AWP\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n","metadata":{"id":"dc38dd05-2f5b-4077-8c47-a28a08210dae","outputId":"66f8f48f-bd39-48df-8d59-09dee35af500","execution":{"iopub.status.busy":"2023-08-27T04:32:11.746356Z","iopub.execute_input":"2023-08-27T04:32:11.74717Z","iopub.status.idle":"2023-08-27T04:33:05.113929Z","shell.execute_reply.started":"2023-08-27T04:32:11.747133Z","shell.execute_reply":"2023-08-27T04:33:05.113013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install cached-property\nfrom cached_property import cached_property\nfrom shutil import copyfile\n\n!pip install fastparquet\nimport fastparquet\n\n!pip install Levenshtein\nimport Levenshtein as lev\nstarttime = time.time()","metadata":{"id":"e57e922c-9237-49bf-b245-fcf15b74b226","outputId":"baf063ba-b69e-434d-d812-9ed6c878cabd","execution":{"iopub.status.busy":"2023-08-27T04:33:05.115679Z","iopub.execute_input":"2023-08-27T04:33:05.116155Z","iopub.status.idle":"2023-08-27T04:33:23.906076Z","shell.execute_reply.started":"2023-08-27T04:33:05.116121Z","shell.execute_reply":"2023-08-27T04:33:23.904925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SEED","metadata":{"id":"o7wgdyCAlsMw"}},{"cell_type":"code","source":"import random\nSEED = 42\nrandom.seed(SEED)       # Set seed for Python's random module\nnp.random.seed(SEED)    # Set seed for numpy\ntf.random.set_seed(SEED) # Set seed for TensorFlow","metadata":{"id":"PiiWczbcluZe","execution":{"iopub.status.busy":"2023-08-27T04:33:23.907558Z","iopub.execute_input":"2023-08-27T04:33:23.907878Z","iopub.status.idle":"2023-08-27T04:33:23.913779Z","shell.execute_reply.started":"2023-08-27T04:33:23.907848Z","shell.execute_reply":"2023-08-27T04:33:23.912902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU setup for Colab TPU","metadata":{"id":"2JaSwXrZTbgS"}},{"cell_type":"code","source":"print(\"Tensorflow version \" + tf.__version__)\n\ntry:\n  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n  # strategy = tf.distribute.TPUStrategy(tpu)\n  print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\n  # print('REPLICA: ', strategy.num_replicas_in_sync)\nexcept ValueError:\n  raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n# strategy = tf.distribute.TPUStrategy(tpu)\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.TPUStrategy(tpu)\nprint('REPLICA: ', strategy.num_replicas_in_sync)","metadata":{"id":"GpY-QwAJTc3d","outputId":"28fbfecc-e576-4893-b72f-fa952263b32d","execution":{"iopub.status.busy":"2023-08-27T04:34:19.432398Z","iopub.execute_input":"2023-08-27T04:34:19.433386Z","iopub.status.idle":"2023-08-27T04:34:19.437909Z","shell.execute_reply.started":"2023-08-27T04:34:19.43333Z","shell.execute_reply":"2023-08-27T04:34:19.436845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU setup for Kaggle TPU","metadata":{}},{"cell_type":"code","source":"# # https://www.kaggle.com/code/shlomoron/aslfr-ctc-on-tpu\n# # Configure Strategy. Assume TPU...if not set default for GPU\n# tpu = None\n# try:\n#     tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=\"local\") # \"local\" for 1VM TPU\n#     strategy = tf.distribute.TPUStrategy(tpu)\n#     print(\"on TPU\")\n#     print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n# except:\n#     strategy = tf.distribute.get_strategy()","metadata":{"id":"JxonR5TXTe4-","outputId":"f5328a89-4815-4257-904a-fb55c14a7849","execution":{"iopub.status.busy":"2023-08-27T04:35:02.838106Z","iopub.execute_input":"2023-08-27T04:35:02.838617Z","iopub.status.idle":"2023-08-27T04:35:11.260129Z","shell.execute_reply.started":"2023-08-27T04:35:02.838583Z","shell.execute_reply":"2023-08-27T04:35:11.259151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/shlomoron/aslfr-ctc-on-tpu\n##Uncommand this cell for Kaggle TPU\n## copy our file into the working directory (make sure it has .py suffix)\n# copyfile(src = \"/kaggle/input/ctc-tpu/CTC_TPU.py\", dst = \"/kaggle/working//CTC_TPU.py\")\n\n# # import all our functions\n# from CTC_TPU import classic_ctc_loss","metadata":{"execution":{"iopub.status.busy":"2023-08-27T04:35:17.084301Z","iopub.execute_input":"2023-08-27T04:35:17.084692Z","iopub.status.idle":"2023-08-27T04:35:17.110284Z","shell.execute_reply.started":"2023-08-27T04:35:17.084663Z","shell.execute_reply":"2023-08-27T04:35:17.109298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# USE Supply or Val","metadata":{"id":"vw7yF2rfBNYY"}},{"cell_type":"code","source":"USE_VAL = True\nUSE_SUPPLY = False","metadata":{"id":"akBJsB7cBMPQ","execution":{"iopub.status.busy":"2023-08-27T04:35:19.068752Z","iopub.execute_input":"2023-08-27T04:35:19.069167Z","iopub.status.idle":"2023-08-27T04:35:19.073815Z","shell.execute_reply.started":"2023-08-27T04:35:19.069117Z","shell.execute_reply":"2023-08-27T04:35:19.07273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GCS_PATH","metadata":{"id":"eBpqBBedTYyW"}},{"cell_type":"code","source":"GCS_PATH = {'ASL': 'gs://kds-a7bec0f3be4fe8073d283fc537263ae5223e9bbd2d812802cbcaf753',\n            'new-coordinate-mean-std': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5',\n            '5%-val-dataset': 'gs://kds-cfc32c8b51aa4053dc15c14ce0c260e474a660f82290940026a6c92b',\n            '5%-train-dataset': 'gs://kds-309ee62af4d36117b93b96f5a8fcebdea475dcb64a7f895b077c2c95',\n            '1%-val-dataset': 'gs://kds-5bded0c2ff71d0dd2f7d6306f2940737b2952ca4b6b7bf6864931eab',\n            '1%-train-dataset': 'gs://kds-71da91630e0537bc8c375feb45e47971b0494b30ceb83d5990205c9b',\n            'supply-pid': 'gs://kds-ca690717e0cb72e6e077d6c1c91998e28e234c7ab6e33f58c4665a1b',\n            'supply-mean-std': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116',\n\n      }\n# load supply set\nTRAIN_SUPPLY = tf.io.gfile.glob(GCS_PATH['supply-pid']+'/*.tfrecord')\n\n# load train and val set, you can chose 95%/5% or 99%/1% spilt set \nVAL_FILENAMES =  tf.io.gfile.glob(GCS_PATH['1%-val-dataset']+'/*.tfrecord')\nTRAIN_FILENAMES = tf.io.gfile.glob(GCS_PATH['1%-train-dataset']+'/*.tfrecord')\n\nprint(len(TRAIN_FILENAMES),len(VAL_FILENAMES))\nprint(len(TRAIN_SUPPLY))\n\n\nCOMPETITION_PATH = GCS_PATH['ASL']\n!gsutil cp {COMPETITION_PATH}/train.csv .\n!gsutil cp {COMPETITION_PATH}/character_to_prediction_index.json .\n#!gsutil ls gs://kds-cd89af783a43be44bd46c8bb80046db59bf92ffaf527609ca13d24b8","metadata":{"id":"FINtnOaWOpn6","outputId":"642abee2-8c1d-4f46-d720-8b63bcd208f6","execution":{"iopub.status.busy":"2023-08-27T04:35:21.071491Z","iopub.execute_input":"2023-08-27T04:35:21.071892Z","iopub.status.idle":"2023-08-27T04:35:23.847921Z","shell.execute_reply.started":"2023-08-27T04:35:21.07186Z","shell.execute_reply":"2023-08-27T04:35:23.846274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setup frame and coordinate","metadata":{"id":"Vba75PAInPoi"}},{"cell_type":"code","source":"# #For Kaggle TPU\n# with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n#     char_to_num = json.load(f)\n    \nwith open (\"/content/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\npad_token = '^'\npad_token_idx = 59\n\nchar_to_num[pad_token] = pad_token_idx\n\nnum_to_char = {j:i for i,j in char_to_num.items()}\n\n#df = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\ndf = pd.read_csv('train.csv')\n\n\nLIP = [\n    61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\n# new face_id\nface_id = [454,356,323,361,389,288,251,264,447,366,368,\n           401,397,435,284,301,372,345,383,367,365,352,433,\n           376,298,265,93,234,300,132,340,353,127]\n# LIP = list(set(LIP + face_id))\nfor k in face_id:\n    if k not in LIP:\n        LIP.append(k)\nl = len(LIP)\nprint(len(LIP))\nLIP  = LIP[:int(l-l/4)]\n# LPOSE = [13, 15, 17, 19, 21]\n# RPOSE = [14, 16, 18, 20, 22]\nLPOSE = [1,3,5,7,9,11,13, 15,17, 19, 21,23,25,27,29,31]#\nRPOSE = [0,2,4,6,8,10,12, 14,16, 18, 20,22,24,26,28,30]#\nPOSE = LPOSE + RPOSE\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE] + [f'x_face_{i}' for i in LIP]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE] + [f'y_face_{i}' for i in LIP]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE] + [f'z_face_{i}' for i in LIP]\n\nSEL_COLS = X + Y + Z\nFRAME_LEN = 256#128\nMAX_PHRASE_LENGTH = 64\n\nLIP_IDX_X   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"x\" in col]\nRHAND_IDX_X = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"x\" in col]\nLHAND_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"x\" in col]\nRPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in RPOSE and \"x\" in col]\nLPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in LPOSE and \"x\" in col]\n\nLIP_IDX_Y   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"y\" in col]\nRHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"y\" in col]\nLHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"y\" in col]\nRPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in RPOSE and \"y\" in col]\nLPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in LPOSE and \"y\" in col]\n\nLIP_IDX_Z   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"z\" in col]\nRHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"z\" in col]\nLHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"z\" in col]\nRPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in RPOSE and \"z\" in col]\nLPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col.split('_')[-1]) in LPOSE and \"z\" in col]\n\n# check path for supply-mean-std\nif USE_SUPPLY:\n  paths = {\n      'RHM': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116/rh_mean.npy',\n      'LHM': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116/lh_mean.npy',\n      'RPM': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116/rp_mean.npy',\n      'LPM': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116/lp_mean.npy',\n      'LIPM': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116/lip_mean.npy',\n      'RHS': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116/rh_std.npy',\n      'LHS': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116/lh_std.npy',\n      'RPS': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116/rp_std.npy',\n      'LPS': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116/lp_std.npy',\n      'LIPS': 'gs://kds-ab9973aa093552472b53710e8b9bcaac6ed07fe79ca397062e266116/lip_std.npy'\n  }\n\n  loaded_data = {key: np.load(tf.io.gfile.GFile(path, 'rb')) for key, path in paths.items()}\n\n  RHM = loaded_data['RHM']\n  LHM = loaded_data['LHM']\n  RPM = loaded_data['RPM']\n  LPM = loaded_data['LPM']\n  LIPM = loaded_data['LIPM']\n\n  RHS = loaded_data['RHS']\n  LHS = loaded_data['LHS']\n  RPS = loaded_data['RPS']\n  LPS = loaded_data['LPS']\n  LIPS = loaded_data['LIPS']\n\n# check path for new-coordinate-mean-std   \nif USE_VAL:\n  paths = {\n      'RHM': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5/rh_mean.npy',\n      'LHM': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5/lh_mean.npy',\n      'RPM': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5/rp_mean.npy',\n      'LPM': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5/lp_mean.npy',\n      'LIPM': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5/lip_mean.npy',\n      'RHS': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5/rh_std.npy',\n      'LHS': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5/lh_std.npy',\n      'RPS': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5/rp_std.npy',\n      'LPS': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5/lp_std.npy',\n      'LIPS': 'gs://kds-eccfa04ab4e08b5b48389f4abb31fb092c6cfd0394610d8929a376b5/lip_std.npy'\n  }\n\n  loaded_data = {key: np.load(tf.io.gfile.GFile(path, 'rb')) for key, path in paths.items()}\n\n  RHM = loaded_data['RHM']\n  LHM = loaded_data['LHM']\n  RPM = loaded_data['RPM']\n  LPM = loaded_data['LPM']\n  LIPM = loaded_data['LIPM']\n\n  RHS = loaded_data['RHS']\n  LHS = loaded_data['LHS']\n  RPS = loaded_data['RPS']\n  LPS = loaded_data['LPS']\n  LIPS = loaded_data['LIPS']","metadata":{"id":"cc07ff1f-881e-47f8-ae39-8dec0b6e6537","outputId":"5daf6111-fb68-4dc9-f25f-5651b17b64fb","execution":{"iopub.status.busy":"2023-08-27T04:35:50.659453Z","iopub.execute_input":"2023-08-27T04:35:50.65984Z","iopub.status.idle":"2023-08-27T04:35:58.633888Z","shell.execute_reply.started":"2023-08-27T04:35:50.65981Z","shell.execute_reply":"2023-08-27T04:35:58.632715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Proprocessing","metadata":{"id":"N2mWkeVmlq3F"}},{"cell_type":"code","source":"\n@tf.function()\ndef resize_pad(x):\n    if tf.shape(x)[0] < FRAME_LEN:\n        x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]), constant_values=float(\"NaN\"))\n    else:\n        x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n    return x\n\n@tf.function(jit_compile=True)\ndef pre_process0(x):\n    lip_x = tf.gather(x, LIP_IDX_X, axis=1)\n    lip_y = tf.gather(x, LIP_IDX_Y, axis=1)\n    lip_z = tf.gather(x, LIP_IDX_Z, axis=1)\n\n    rhand_x = tf.gather(x, RHAND_IDX_X, axis=1)\n    rhand_y = tf.gather(x, RHAND_IDX_Y, axis=1)\n    rhand_z = tf.gather(x, RHAND_IDX_Z, axis=1)\n\n    lhand_x = tf.gather(x, LHAND_IDX_X, axis=1)\n    lhand_y = tf.gather(x, LHAND_IDX_Y, axis=1)\n    lhand_z = tf.gather(x, LHAND_IDX_Z, axis=1)\n\n    rpose_x = tf.gather(x, RPOSE_IDX_X, axis=1)\n    rpose_y = tf.gather(x, RPOSE_IDX_Y, axis=1)\n    rpose_z = tf.gather(x, RPOSE_IDX_Z, axis=1)\n\n    lpose_x = tf.gather(x, LPOSE_IDX_X, axis=1)\n    lpose_y = tf.gather(x, LPOSE_IDX_Y, axis=1)\n    lpose_z = tf.gather(x, LPOSE_IDX_Z, axis=1)\n\n    lip   = tf.concat([lip_x[..., tf.newaxis], lip_y[..., tf.newaxis], lip_z[..., tf.newaxis]], axis=-1)\n    rhand = tf.concat([rhand_x[..., tf.newaxis], rhand_y[..., tf.newaxis], rhand_z[..., tf.newaxis]], axis=-1)\n    lhand = tf.concat([lhand_x[..., tf.newaxis], lhand_y[..., tf.newaxis], lhand_z[..., tf.newaxis]], axis=-1)\n    rpose = tf.concat([rpose_x[..., tf.newaxis], rpose_y[..., tf.newaxis], rpose_z[..., tf.newaxis]], axis=-1)\n    lpose = tf.concat([lpose_x[..., tf.newaxis], lpose_y[..., tf.newaxis], lpose_z[..., tf.newaxis]], axis=-1)\n\n    hand = tf.concat([rhand, lhand,lip,rpose,lpose], axis=1)\n    hand = tf.where(tf.math.is_nan(hand), 0.0, hand)\n    mask = tf.math.not_equal(tf.reduce_sum(hand, axis=[1, 2]), 0.0)\n\n    lip = lip[mask]\n    rhand = rhand[mask]\n    lhand = lhand[mask]\n    rpose = rpose[mask]\n    lpose = lpose[mask]\n\n    return lip, rhand, lhand, rpose, lpose\n\n@tf.function()\ndef pre_process1(lip, rhand, lhand, rpose, lpose):\n    lip   = (resize_pad(lip) - LIPM) / LIPS\n    rhand = (resize_pad(rhand) - RHM) / RHS\n    lhand = (resize_pad(lhand) - LHM) / LHS\n    rpose = (resize_pad(rpose) - RPM) / RPS\n    lpose = (resize_pad(lpose) - LPM) / LPS\n\n    x = tf.concat([lip, rhand, lhand, rpose, lpose], axis=1)\n    ##motionfeature\n    length = tf.shape(x)[0]\n    dx = tf.cond(tf.shape(x)[0]>1,lambda:tf.pad(x[1:] - x[:-1], [[0,1],[0,0],[0,0]]),lambda:tf.zeros_like(x))\n    dx2 = tf.cond(tf.shape(x)[0]>2,lambda:tf.pad(x[2:] - x[:-2], [[0,2],[0,0],[0,0]]),lambda:tf.zeros_like(x))\n    x = tf.concat([x,dx,dx2], axis = -1)\n    s = tf.shape(x)\n    x = tf.reshape(x, (s[0], s[1]*s[2]))\n\n    x = tf.where(tf.math.is_nan(x), 0.0, x)\n    return x\n\nINPUT_SHAPE = [256,1152]\n\n# pre0 = pre_process0(frames)\n# pre1 = pre_process1(*pre0)\n# INPUT_SHAPE = list(pre1.shape)\n# print(INPUT_SHAPE)\n# pre1","metadata":{"id":"f8a7b72e-99ec-4f8a-b7fb-eccd8cb9113c","execution":{"iopub.status.busy":"2023-08-27T04:36:13.361406Z","iopub.execute_input":"2023-08-27T04:36:13.362259Z","iopub.status.idle":"2023-08-27T04:36:13.392539Z","shell.execute_reply.started":"2023-08-27T04:36:13.362224Z","shell.execute_reply":"2023-08-27T04:36:13.391542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_fn(record_bytes):\n    schema = {\n        \"lip\": tf.io.VarLenFeature(tf.float32),\n        \"rhand\": tf.io.VarLenFeature(tf.float32),\n        \"lhand\": tf.io.VarLenFeature(tf.float32),\n        \"rpose\": tf.io.VarLenFeature(tf.float32),\n        \"lpose\": tf.io.VarLenFeature(tf.float32),\n        \"phrase\": tf.io.VarLenFeature(tf.int64)\n    }\n    x = tf.io.parse_single_example(record_bytes, schema)\n\n    lip = tf.reshape(tf.sparse.to_dense(x[\"lip\"]), (-1, 54, 3))\n    rhand = tf.reshape(tf.sparse.to_dense(x[\"rhand\"]), (-1, 21, 3))\n    lhand = tf.reshape(tf.sparse.to_dense(x[\"lhand\"]), (-1, 21, 3))\n    rpose = tf.reshape(tf.sparse.to_dense(x[\"rpose\"]), (-1, 16, 3))\n    lpose = tf.reshape(tf.sparse.to_dense(x[\"lpose\"]), (-1, 16, 3))\n    phrase = tf.sparse.to_dense(x[\"phrase\"])\n\n    return lip, rhand, lhand, rpose, lpose, phrase\n\ndef pre_process_fn(lip, rhand, lhand, rpose, lpose, phrase):\n    phrase = tf.pad(phrase, [[0, MAX_PHRASE_LENGTH-tf.shape(phrase)[0]]], constant_values=pad_token_idx)\n    return pre_process1(lip, rhand, lhand, rpose, lpose), phrase\n\n","metadata":{"id":"a80b53eb-3079-4de5-b081-ebef60364f76","execution":{"iopub.status.busy":"2023-08-27T04:36:15.261171Z","iopub.execute_input":"2023-08-27T04:36:15.261548Z","iopub.status.idle":"2023-08-27T04:36:15.27394Z","shell.execute_reply.started":"2023-08-27T04:36:15.261519Z","shell.execute_reply":"2023-08-27T04:36:15.273024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Random drop frame Augmentation","metadata":{"id":"BlEpHdyNnJIB"}},{"cell_type":"code","source":"def random_zero_out_sequence(sequence):\n    \"\"\"\n    Set the entire sequence to zero.\n    \"\"\"\n    return tf.zeros_like(sequence)\n\ndef decode_fn_with_random_drop(lip, rhand, lhand, rpose, lpose, drop_prob=0.025):\n\n    # Choose one of the sequences randomly based on the drop probability\n    drop_choice = tf.cond(tf.random.uniform(()) < drop_prob,\n                          lambda: tf.random.uniform((), 0, 5, dtype=tf.int32),\n                          lambda: tf.constant(-1, dtype=tf.int32))\n\n    lip = tf.cond(tf.equal(drop_choice, 0), lambda: random_zero_out_sequence(lip), lambda: lip)\n    rhand = tf.cond(tf.equal(drop_choice, 1), lambda: random_zero_out_sequence(rhand), lambda: rhand)\n    lhand = tf.cond(tf.equal(drop_choice, 2), lambda: random_zero_out_sequence(lhand), lambda: lhand)\n    rpose = tf.cond(tf.equal(drop_choice, 3), lambda: random_zero_out_sequence(rpose), lambda: rpose)\n    lpose = tf.cond(tf.equal(drop_choice, 4), lambda: random_zero_out_sequence(lpose), lambda: lpose)\n\n    return lip, rhand, lhand, rpose, lpose\n\ndef spatial_random_affine(xyz,\n    scale  = (0.9,1.1),\n    shear = (-0.1,0.1),\n    shift  = (-0.05,0.05),\n    degree = (-10,10),\n):\n    center = tf.constant([0.5,0.5])\n    if scale is not None:\n        scale = tf.random.uniform((),*scale)\n        xyz = scale*xyz\n\n    if shear is not None:\n        xy = xyz[...,:2]\n        z = xyz[...,2:]\n        shear_x = shear_y = tf.random.uniform((),*shear)\n        if tf.random.uniform(()) < 0.5:\n            shear_x = 0.\n        else:\n            shear_y = 0.\n        shear_mat = tf.identity([\n            [1.,shear_x],\n            [shear_y,1.]\n        ])\n        xy = xy @ shear_mat\n        center = center + [shear_y, shear_x]\n        xyz = tf.concat([xy,z], axis=-1)\n\n    if degree is not None:\n        xy = xyz[...,:2]\n        z = xyz[...,2:]\n        xy -= center\n        degree = tf.random.uniform((),*degree)\n        radian = degree/180*np.pi\n        c = tf.math.cos(radian)\n        s = tf.math.sin(radian)\n        rotate_mat = tf.identity([\n            [c, s],\n            [-s, c],\n        ])\n        xy = xy @ rotate_mat\n        xy = xy + center\n        xyz = tf.concat([xy,z], axis=-1)\n\n    if shift is not None:\n        shift = tf.random.uniform((),*shift)\n        xyz = xyz + shift\n\n    return xyz\n\ndef augmented_fn(lip, rhand, lhand, rpose, lpose, phrase):\n\n    # Apply spatial random affine\n    if tf.random.uniform(()) < 0.9:\n        features = tf.concat([lip, rhand, lhand, rpose, lpose], axis=1)\n        features = spatial_random_affine(features)\n        lip, rhand, lhand, rpose, lpose = tf.split(features, axis=1, num_or_size_splits=[54, 21, 21, 16, 16])\n\n    # Apply the random drop augmentation\n    lip, rhand, lhand, rpose, lpose = decode_fn_with_random_drop(lip, rhand, lhand, rpose, lpose)\n\n    return lip, rhand, lhand, rpose, lpose, phrase","metadata":{"id":"pRyIm_JIZY0h","execution":{"iopub.status.busy":"2023-08-27T04:36:16.609647Z","iopub.execute_input":"2023-08-27T04:36:16.610023Z","iopub.status.idle":"2023-08-27T04:36:16.634701Z","shell.execute_reply.started":"2023-08-27T04:36:16.609992Z","shell.execute_reply":"2023-08-27T04:36:16.6337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and Val_Set","metadata":{"id":"sIlGLmUXlzqI"}},{"cell_type":"code","source":"train_batch_size = 32\nval_batch_size = 32\n\nsupply_tffiles = TRAIN_SUPPLY\nval_tffiles = VAL_FILENAMES\ntrain_tffiles = TRAIN_FILENAMES\n\nrandom.seed(SEED)\nrandom.shuffle(val_tffiles)\nrandom.shuffle(train_tffiles)\nrandom.shuffle(supply_tffiles)\n\n\n# Trianing on supplyment dataset\n\nif USE_SUPPLY:\n  supply_dataset_pre = (tf.data.TFRecordDataset(supply_tffiles)\n                            .prefetch(tf.data.AUTOTUNE)\n                            .map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE)\n                            .map(pre_process_fn, num_parallel_calls=tf.data.AUTOTUNE))\n\n# Use Validation set\nif USE_VAL:\n  val_dataset_pre =  (tf.data.TFRecordDataset(val_tffiles)\n                          .prefetch(tf.data.AUTOTUNE)\n                          .map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE)\n                          .map(pre_process_fn, num_parallel_calls=tf.data.AUTOTUNE))\n\n  train_dataset_pre =  (tf.data.TFRecordDataset(train_tffiles)\n                          .prefetch(tf.data.AUTOTUNE)\n                          .map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE)\n                          .map(pre_process_fn, num_parallel_calls=tf.data.AUTOTUNE))\n\n","metadata":{"id":"5TPWXYrVUnzu","execution":{"iopub.status.busy":"2023-08-27T04:36:19.134419Z","iopub.execute_input":"2023-08-27T04:36:19.134788Z","iopub.status.idle":"2023-08-27T04:36:20.109257Z","shell.execute_reply.started":"2023-08-27T04:36:19.134758Z","shell.execute_reply":"2023-08-27T04:36:20.108115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate val and train dataset","metadata":{"id":"BAxpeVpOhILt"}},{"cell_type":"code","source":"if USE_SUPPLY:\n  train_supply_items = [x for x in supply_dataset_pre]\n  train_supply_items_X = [x[0] for x in train_supply_items]\n  train_supply_items_y = [tf.cast(x[1], dtype = tf.int32) for x in train_supply_items]\n  train_supply_dataset = tf.data.Dataset.from_tensor_slices((train_supply_items_X, train_supply_items_y)).prefetch(tf.data.AUTOTUNE).shuffle(60000).batch(train_batch_size, drop_remainder=True).prefetch(tf.data.AUTOTUNE)\n  batch = next(iter(train_supply_dataset))\n  print(len(train_supply_dataset))\n\nif USE_VAL:\n  val_items = [x for x in val_dataset_pre]\n  val_items_X = [x[0] for x in val_items]\n  val_items_y = [tf.cast(x[1], dtype = tf.int32) for x in val_items]\n  val_dataset = tf.data.Dataset.from_tensor_slices((val_items_X, val_items_y)).prefetch(tf.data.AUTOTUNE).batch(val_batch_size, drop_remainder=True).prefetch(tf.data.AUTOTUNE)\n\n  train_items = [x for x in train_dataset_pre]\n  train_items_X = [x[0] for x in train_items]\n  train_items_y = [tf.cast(x[1], dtype = tf.int32) for x in train_items]\n  train_dataset = tf.data.Dataset.from_tensor_slices((train_items_X, train_items_y)).prefetch(tf.data.AUTOTUNE).shuffle(60000).batch(train_batch_size, drop_remainder=True).prefetch(tf.data.AUTOTUNE)\n  batch = next(iter(val_dataset))\n  print(batch[0].shape, batch[1].shape)\n  print(len(train_dataset))\n  print(len(val_dataset))\n  nums_epoch_train = len(train_dataset)","metadata":{"id":"0954b8d6-ec86-4a25-ad87-325d12b9cc07","outputId":"75941a52-6e55-42e2-b48c-5cbe4be9ee66","execution":{"iopub.status.busy":"2023-08-27T04:36:33.456057Z","iopub.execute_input":"2023-08-27T04:36:33.456441Z","iopub.status.idle":"2023-08-27T04:45:57.252602Z","shell.execute_reply.started":"2023-08-27T04:36:33.456409Z","shell.execute_reply":"2023-08-27T04:45:57.251403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Config","metadata":{"id":"JXtgj11jUzaM"}},{"cell_type":"code","source":"class ECA(tf.keras.layers.Layer):\n    def __init__(self, kernel_size=5, **kwargs):\n        super().__init__(**kwargs)\n        self.supports_masking = True\n        self.kernel_size = kernel_size\n        self.conv = tf.keras.layers.Conv1D(1, kernel_size=kernel_size, strides=1, padding=\"same\", use_bias=False)\n\n    def call(self, inputs, mask=None):\n        nn = tf.keras.layers.GlobalAveragePooling1D()(inputs, mask=mask)\n        nn = tf.expand_dims(nn, -1)\n        nn = self.conv(nn)\n        nn = tf.squeeze(nn, -1)\n        nn = tf.nn.sigmoid(nn)\n        nn = nn[:,None,:]\n        return inputs * nn\n\nclass CausalDWConv1D(tf.keras.layers.Layer):\n    def __init__(self,\n        kernel_size=17,\n        dilation_rate=1,\n        use_bias=False,\n        depthwise_initializer='glorot_uniform',\n        name='', **kwargs):\n        super().__init__(name=name,**kwargs)\n        self.causal_pad = tf.keras.layers.ZeroPadding1D((dilation_rate*(kernel_size-1),0),name=name + '_pad')\n        self.dw_conv = tf.keras.layers.DepthwiseConv1D(\n                            kernel_size,\n                            strides=1,\n                            dilation_rate=dilation_rate,\n                            padding='valid',\n                            use_bias=use_bias,\n                            depthwise_initializer=depthwise_initializer,\n                            name=name + '_dwconv')\n        self.supports_masking = True\n\n    def call(self, inputs):\n        x = self.causal_pad(inputs)\n        x = self.dw_conv(x)\n        return x\n\ndef Conv1DBlock(channel_size,\n          kernel_size,\n          dilation_rate=1,\n          drop_rate=0.0,\n          expand_ratio=2,\n          se_ratio=0.25,\n          activation='swish',\n          name=None):\n    '''\n    efficient conv1d block, @hoyso48\n    '''\n    if name is None:\n        name = str(tf.keras.backend.get_uid(\"mbblock\"))\n    # Expansion phase\n    def apply(inputs):\n        channels_in = tf.keras.backend.int_shape(inputs)[-1]\n        channels_expand = channels_in * expand_ratio\n\n        skip = inputs\n\n        x = tf.keras.layers.Dense(\n            channels_expand,\n            use_bias=True,\n            activation=activation,\n            name=name + '_expand_conv')(inputs)\n\n        # Depthwise Convolution\n        x = CausalDWConv1D(kernel_size,\n            dilation_rate=dilation_rate,\n            use_bias=False,\n            name=name + '_dwconv')(x)\n\n        x = tf.keras.layers.BatchNormalization(momentum=0.95, name=name + '_bn')(x)\n\n        x  = ECA()(x)\n\n        x = tf.keras.layers.Dense(\n            channel_size,\n            use_bias=True,\n            name=name + '_project_conv')(x)\n\n        if drop_rate > 0:\n            x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1), name=name + '_drop')(x)\n\n        #保留残差网络\n        # if (channels_in == channel_size):\n        x = tf.keras.layers.add([x, skip], name=name + '_add')\n        return x\n\n    return apply\n\nclass MultiHeadSelfAttention(tf.keras.layers.Layer):\n    def __init__(self, dim=256, num_heads=4, dropout=0, **kwargs):\n        super().__init__(**kwargs)\n        self.dim = dim\n        self.scale = self.dim ** -0.5\n        self.num_heads = num_heads\n        self.qkv = tf.keras.layers.Dense(3 * dim, use_bias=False)\n        self.drop1 = tf.keras.layers.Dropout(dropout)\n        self.proj = tf.keras.layers.Dense(dim, use_bias=False)\n        self.supports_masking = True\n\n    def call(self, inputs, mask=None):\n        qkv = self.qkv(inputs)\n        qkv = tf.keras.layers.Permute((2, 1, 3))(tf.keras.layers.Reshape((-1, self.num_heads, self.dim * 3 // self.num_heads))(qkv))\n        q, k, v = tf.split(qkv, [self.dim // self.num_heads] * 3, axis=-1)\n\n        attn = tf.matmul(q, k, transpose_b=True) * self.scale\n\n        if mask is not None:\n            mask = mask[:, None, None, :]\n\n        attn = tf.keras.layers.Softmax(axis=-1)(attn, mask=mask)\n        attn = self.drop1(attn)\n\n        x = attn @ v\n        x = tf.keras.layers.Reshape((-1, self.dim))(tf.keras.layers.Permute((2, 1, 3))(x))\n        x = self.proj(x)\n        return x\n\n\ndef TransformerBlock(dim=256, num_heads=6, expand=4, attn_dropout=0.2, drop_rate=0.2, activation='swish'):\n    def apply(inputs):\n        x = inputs\n        x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n        x = MultiHeadSelfAttention(dim=dim,num_heads=num_heads,dropout=attn_dropout)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([inputs, x])\n        attn_out = x\n\n        x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n        x = tf.keras.layers.Dense(dim*expand, use_bias=False, activation=activation)(x)\n        x = tf.keras.layers.Dense(dim, use_bias=False)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([attn_out, x])\n        return x\n    return apply\n\ndef positional_encoding(maxlen, num_hid):\n        depth = num_hid/2\n        positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n        depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n        angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n        angle_rads = tf.linalg.matmul(positions, angle_rates)\n        pos_encoding = tf.concat(\n          [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n          axis=-1)\n        return pos_encoding","metadata":{"id":"5188bbf5-1bef-4085-9699-0db35ee380fd","execution":{"iopub.status.busy":"2023-08-27T04:46:22.028768Z","iopub.execute_input":"2023-08-27T04:46:22.02977Z","iopub.status.idle":"2023-08-27T04:46:22.070705Z","shell.execute_reply.started":"2023-08-27T04:46:22.029725Z","shell.execute_reply":"2023-08-27T04:46:22.069642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu\n\nLHAND_IDX = LHAND_IDX_X + LHAND_IDX_Y + LHAND_IDX_Z\nRHAND_IDX = RHAND_IDX_X + RHAND_IDX_Y + RHAND_IDX_Z\nLIP_IDX = LIP_IDX_X + LIP_IDX_Y + LIP_IDX_Z\nPOSE_IDX = LPOSE_IDX_X + RPOSE_IDX_X + LPOSE_IDX_Y + RPOSE_IDX_Y + LPOSE_IDX_Z + RPOSE_IDX_Z\n\n# Landmark Embedding\n# Embeds a landmark using local features and global features\nclass LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name='landmark_embedding'):\n        super(LandmarkEmbedding, self).__init__(name=name)\n        self.units = units\n        self.supports_masking = True\n\n    def build(self, input_shape):\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # local feature extractor\n        self.lefthand_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_lefthand', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU)\n        self.righthand_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_righthand', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU)\n        self.lips_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_lips', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU)\n        self.pose_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_pose', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU)\n\n        # full feature extractor\n        self.full_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_full', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU)\n\n        # global feature extractor\n        self.global_mlp = tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_global', use_bias=False, kernel_initializer=INIT_HE_UNIFORM)\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.global_mlp(tf.concat([self.full_extractor(x), self.local_extractor(x)], axis=-1))\n            )\n\n    def local_extractor(self, x):\n\n        lefthand_feature = self.lefthand_mlp(tf.gather(x, LHAND_IDX, axis=-1))\n        righthad_featrue = self.righthand_mlp(tf.gather(x, RHAND_IDX, axis=-1))\n        # Keeps dominant hand feature\n        hand_feature = tf.reduce_max(tf.stack([lefthand_feature, righthad_featrue], axis=-1), axis=-1)\n        lips_featrue = self.lips_mlp(tf.gather(x, LIP_IDX, axis=-1))\n        pose_featrue = self.pose_mlp(tf.gather(x, POSE_IDX, axis=-1))\n\n        return tf.concat([hand_feature, lips_featrue, pose_featrue], axis=-1)\n\n    def full_extractor(self, x):\n        return self.full_mlp(x)","metadata":{"id":"6d4eecfe","execution":{"iopub.status.busy":"2023-08-27T04:46:23.443853Z","iopub.execute_input":"2023-08-27T04:46:23.444759Z","iopub.status.idle":"2023-08-27T04:46:23.464956Z","shell.execute_reply.started":"2023-08-27T04:46:23.444725Z","shell.execute_reply":"2023-08-27T04:46:23.463848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CTCLOSS with label smoothing","metadata":{"id":"mutnC_zLrRX6"}},{"cell_type":"code","source":"def CTCLoss(labels, logits):\n    label_length = tf.reduce_sum(tf.cast(labels != pad_token_idx, tf.int32), axis=-1)\n    logit_length = tf.ones(tf.shape(logits)[0], dtype=tf.int32) * tf.shape(logits)[1]\n    loss = tf.nn.ctc_loss(\n            labels=labels,\n            logits=logits,\n            label_length=label_length,\n            logit_length=logit_length,\n            blank_index=pad_token_idx,\n            logits_time_major=False\n        )\n    loss = tf.reduce_mean(loss)\n    return loss","metadata":{"id":"f5beddf1-ff67-4b5a-8d5f-b0bc6f846162","execution":{"iopub.status.busy":"2023-08-27T04:46:25.329616Z","iopub.execute_input":"2023-08-27T04:46:25.330023Z","iopub.status.idle":"2023-08-27T04:46:25.337256Z","shell.execute_reply.started":"2023-08-27T04:46:25.329991Z","shell.execute_reply":"2023-08-27T04:46:25.336291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/shlomoron/aslfr-ctc-on-tpu\n# def CTCLoss(labels, logits):\n#     label_length = tf.reduce_sum(tf.cast(labels != pad_token_idx, tf.int32), axis=-1)\n#     logit_length = tf.ones(tf.shape(logits)[0], dtype=tf.int32) * tf.shape(logits)[1]\n    \n#     loss = classic_ctc_loss(\n#             labels=labels,\n#             logits=logits,\n#             label_length=label_length,\n#             logit_length=logit_length,\n#             blank_index=pad_token_idx,\n#         )\n#     '''\n#     loss = tf.nn.ctc_loss(\n#             labels=labels,\n#             logits=logits,\n#             label_length=label_length,\n#             logit_length=logit_length,\n#             blank_index=pad_token_idx,\n#             logits_time_major=False\n#         )\n#     '''\n#     loss = tf.reduce_mean(loss)\n#     return loss","metadata":{"execution":{"iopub.status.busy":"2023-08-27T04:46:28.177951Z","iopub.execute_input":"2023-08-27T04:46:28.178316Z","iopub.status.idle":"2023-08-27T04:46:28.183497Z","shell.execute_reply.started":"2023-08-27T04:46:28.178286Z","shell.execute_reply":"2023-08-27T04:46:28.182395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def smooth_ctc_loss(labels, logits, num_classes = 60 , blank=0, weight=0.7):\n    # Compute CTC Loss\n    label_length = tf.reduce_sum(tf.cast(labels != pad_token_idx, tf.int32), axis=-1)\n    logit_length = tf.ones(tf.shape(logits)[0], dtype=tf.int32) * tf.shape(logits)[1]\n    ctc_loss = tf.nn.ctc_loss(\n        labels=labels,\n        logits=logits,\n        label_length=label_length,\n        logit_length=logit_length,\n        blank_index=pad_token_idx,\n        logits_time_major=False\n    )\n\n    '''\n    Use this for Kaggle TPU\n    ctc_loss = classic_ctc_loss(\n            labels=labels,\n            logits=logits,\n            label_length=label_length,\n            logit_length=logit_length,\n            blank_index=pad_token_idx,\n        )\n    '''    \n    ctc_loss = tf.reduce_mean(ctc_loss)\n\n    # Compute KL Divergence Loss\n    # Compute the logarithm of probabilities. Avoid taking the log of zero by adding a small constant.\n    kl_inp = tf.nn.softmax(logits)\n\n    # Create the target distribution\n    kl_tar = tf.fill(tf.shape(logits), 1. / num_classes)\n\n    # Compute the KL divergence\n    kldiv_loss = (tf.keras.losses.KLDivergence(tf.keras.losses.Reduction.NONE)(kl_tar, kl_inp)\n                 + tf.keras.losses.KLDivergence(tf.keras.losses.Reduction.NONE)(kl_inp, kl_tar))/2.0\n\n    kldiv_loss = tf.reduce_mean(kldiv_loss)\n\n    # Combined Loss\n    loss = (1. - weight) * ctc_loss + weight * kldiv_loss\n    #loss = ctc_loss\n    #loss = kldiv_loss\n    return loss","metadata":{"id":"-gP_q0LArQg8","execution":{"iopub.status.busy":"2023-08-27T04:46:57.193927Z","iopub.execute_input":"2023-08-27T04:46:57.194846Z","iopub.status.idle":"2023-08-27T04:46:57.206951Z","shell.execute_reply.started":"2023-08-27T04:46:57.194802Z","shell.execute_reply":"2023-08-27T04:46:57.205851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Model","metadata":{"id":"uSjQHlCSVBu1"}},{"cell_type":"code","source":"def get_model(dim = 384, num_blocks = 6, drop_rate = 0.4):\n    with strategy.scope():\n      inp = tf.keras.Input(INPUT_SHAPE)\n      x = tf.keras.layers.Masking(mask_value=0.0)(inp)\n      # original\n      x = tf.keras.layers.Dense(dim, use_bias=False, name='stem_conv')(x)\n      # local & global extractor\n      #x = LandmarkEmbedding(dim)(x)\n      pe = tf.cast(positional_encoding(INPUT_SHAPE[0], dim), dtype=x.dtype)\n      x = x + pe\n      x = tf.keras.layers.BatchNormalization(momentum=0.95,name='stem_bn')(x)\n\n      for i in range(num_blocks):\n          x = Conv1DBlock(dim, 11, drop_rate=drop_rate)(x)\n          x = Conv1DBlock(dim, 5, drop_rate=drop_rate)(x)\n          x = Conv1DBlock(dim, 3, drop_rate=drop_rate)(x)\n          x = TransformerBlock(dim, expand=2)(x)\n\n      x = tf.keras.layers.Dense(dim*2,tf.keras.activations.gelu,name='top_conv')(x)\n      x = tf.keras.layers.Dropout(0.4)(x)\n      x = tf.keras.layers.Dense(len(char_to_num),name='classifier')(x)\n\n      model = tf.keras.Model(inp, x)\n\n      loss = smooth_ctc_loss\n\n      # Adam Optimizer\n      optimizer = tfa.optimizers.RectifiedAdam(sma_threshold=4)\n      optimizer = tfa.optimizers.Lookahead(optimizer, sync_period=5)\n\n      model.compile(loss=loss, optimizer=optimizer)\n\n      return model\n\n#steps_per_epoch = 61955//train_batch_size\ntf.keras.backend.clear_session()\nmodel = get_model()\nmodel(batch[0])\nmodel.summary()","metadata":{"id":"2c03cb5e-b55d-45e6-9a19-a37b78bd00db","outputId":"8bf94097-dc57-4477-89d0-544ebe0cffa2","execution":{"iopub.status.busy":"2023-08-27T04:46:59.489088Z","iopub.execute_input":"2023-08-27T04:46:59.489527Z","iopub.status.idle":"2023-08-27T04:47:10.472589Z","shell.execute_reply.started":"2023-08-27T04:46:59.489493Z","shell.execute_reply":"2023-08-27T04:47:10.456832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def num_to_char_fn(y):\n    return [num_to_char.get(x, \"\") for x in y]\n\n@tf.function()\ndef decode_phrase(pred):\n    x = tf.argmax(pred, axis=1)\n    diff = tf.not_equal(x[:-1], x[1:])\n    adjacent_indices = tf.where(diff)[:, 0]\n    x = tf.gather(x, adjacent_indices)\n    mask = x != pad_token_idx\n    x = tf.boolean_mask(x, mask, axis=0)\n    return x\n\n# A utility function to decode the output of the network\ndef decode_batch_predictions(pred):\n    output_text = []\n    for result in pred:\n        result = \"\".join(num_to_char_fn(decode_phrase(result).numpy()))\n        output_text.append(result)\n    return output_text","metadata":{"id":"1f25d672-cb18-4dd2-ab78-825cb7fc147a","execution":{"iopub.status.busy":"2023-08-27T04:47:17.377202Z","iopub.execute_input":"2023-08-27T04:47:17.378453Z","iopub.status.idle":"2023-08-27T04:47:17.388189Z","shell.execute_reply.started":"2023-08-27T04:47:17.3784Z","shell.execute_reply":"2023-08-27T04:47:17.386756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Epoch Learning Rate","metadata":{"id":"Tf0jDpanmttg"}},{"cell_type":"code","source":"N_EPOCHS = 80 if USE_SUPPLY else 120\nN_WARMUP_EPOCHS = 10\nLR_MAX = 1e-3*0.5\nWD_RATIO = 0.05\nWARMUP_METHOD = \"exp\"","metadata":{"id":"KU_eWz2hmxTL","execution":{"iopub.status.busy":"2023-08-27T04:47:18.813307Z","iopub.execute_input":"2023-08-27T04:47:18.813742Z","iopub.status.idle":"2023-08-27T04:47:18.819785Z","shell.execute_reply.started":"2023-08-27T04:47:18.813709Z","shell.execute_reply":"2023-08-27T04:47:18.818515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Callback","metadata":{"id":"7gDoPtZDmZet"}},{"cell_type":"code","source":"from Levenshtein import distance\nif USE_SUPPLY:\n  val_set = [x for x in train_supply_dataset.take(5)]\nelif USE_VAL:\n  val_set = [x for x in val_dataset]\n\n#/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\nwith open (\"/content/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nclass val_lev_callback(tf.keras.callbacks.Callback):\n    def __init__(self):\n        super().__init__()\n    def on_epoch_end(self, epoch: int, logs=None):\n        if epoch < N_EPOCHS-20:\n          return\n        calculate_val_lev()\n\ndef calculate_val_lev():\n    preds = []\n    targets = []\n    scores = []\n    for batch_idx in range(len(val_set)):\n        preds_batch = model.predict(val_set[batch_idx][0], verbose = 0)\n        targets_batch = val_set[batch_idx][1]\n        for pred_idx in range(len(preds_batch)):\n            preds.append(\"\".join([rev_character_map.get(s, \"\") for s in decode_phrase(preds_batch[pred_idx]).numpy()]))\n            targets.append(\"\".join([rev_character_map.get(s, \"\") for s in targets_batch[pred_idx].numpy()]))\n\n    N = [len(phrase) for phrase in targets]\n    lev_dist = [distance(preds[i], targets[i]) for i in range(len(targets))]\n    print('Lev distance: '+str((np.sum(N) - np.sum(lev_dist))/np.sum(N)))\n    for _ in range(8):\n      i = tf.random.uniform(shape=[], minval=0, maxval=len(targets), dtype=tf.int32).numpy()  # Corrected this line\n      print(f'predic: {preds[i]}')\n      print(f'target: {targets[i]}')\n      print(\"-\" * 100)","metadata":{"id":"LKqtLvu_mbnP","outputId":"c5e245d2-5c07-42ce-e974-0c1e3ca11219","execution":{"iopub.status.busy":"2023-08-27T04:47:44.883989Z","iopub.execute_input":"2023-08-27T04:47:44.884469Z","iopub.status.idle":"2023-08-27T04:47:45.010993Z","shell.execute_reply.started":"2023-08-27T04:47:44.884433Z","shell.execute_reply":"2023-08-27T04:47:45.009428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n\n    def on_epoch_begin(self, epoch, logs=None):\n        model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')\n\nclass save_model_callback(tf.keras.callbacks.Callback):\n    def __init__(self):\n        super().__init__()\n    def on_epoch_end(self, epoch: int, logs=None):\n          if (N_EPOCHS-30 <= epoch < N_EPOCHS-10 and (epoch) % 5 == 0) or epoch >= N_EPOCHS-10 :\n            self.model.save_weights(f\"model_epoch_{epoch}.h5\")","metadata":{"id":"Lenw4Wzzmp18","execution":{"iopub.status.busy":"2023-08-27T04:47:46.438083Z","iopub.execute_input":"2023-08-27T04:47:46.439333Z","iopub.status.idle":"2023-08-27T04:47:46.448316Z","shell.execute_reply.started":"2023-08-27T04:47:46.439287Z","shell.execute_reply":"2023-08-27T04:47:46.447172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n\n    if current_step < num_warmup_steps:\n        if WARMUP_METHOD == 'log':\n            return lr_max * 0.10 ** (num_warmup_steps - current_step)\n        else:\n            return lr_max * 2 ** -(num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max\n\ndef plot_lr_schedule(lr_schedule, epochs):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(1, epochs + 1)\n    x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n    plt.xlim([1, epochs])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n\n    # Increase y-limit for better readability\n    plt.ylim([0, max(lr_schedule) * 1.1])\n\n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n\n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n            if x < len(lr_schedule) - 1:\n                if lr_schedule[x - 1] < val:\n                    ha = 'right'\n                else:\n                    ha = 'left'\n            elif x == 0:\n                ha = 'right'\n            else:\n                ha = 'left'\n            plt.plot(x + 1, val, 'o', color='black');\n            offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n            plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n\n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=0)\n\n# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n\n    def on_epoch_begin(self, epoch, logs=None):\n        model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"id":"88b85019-5c8d-4f5e-af79-5e0f11fcd1d7","outputId":"e54e5e38-010b-4053-aed5-4051ccfa6256","execution":{"iopub.status.busy":"2023-08-27T04:48:08.745405Z","iopub.execute_input":"2023-08-27T04:48:08.746326Z","iopub.status.idle":"2023-08-27T04:48:09.819508Z","shell.execute_reply.started":"2023-08-27T04:48:08.746288Z","shell.execute_reply":"2023-08-27T04:48:09.818128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint\n\n# Specify the path where you want to save the model checkpoint\ncheckpoint_path = \"model_checkpoint.h5\"\n\n# Create the checkpoint callback, specifying details such as the metric to monitor, mode, etc.\nmodel_checkpoint_callback = ModelCheckpoint(\n    filepath=checkpoint_path,\n    save_weights_only=True, # Set to False if you want to save the entire model\n    monitor='loss',         # Monitor validation loss. Change this if you want to monitor another metric.\n    mode='min',             # Mode = 'min' ensures that the model is saved only when the monitored metric (validation loss in this case) decreases.\n    save_best_only=True,    # Only save a model if `loss` has improved.\n    verbose=0               # Log when a new best is saved\n)\n\n","metadata":{"id":"M_DQbH2GGmq5","execution":{"iopub.status.busy":"2023-08-27T04:48:12.701738Z","iopub.execute_input":"2023-08-27T04:48:12.702818Z","iopub.status.idle":"2023-08-27T04:48:12.708925Z","shell.execute_reply.started":"2023-08-27T04:48:12.702778Z","shell.execute_reply":"2023-08-27T04:48:12.707776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training on Supply","metadata":{"id":"fdep5nicmsfg"}},{"cell_type":"code","source":"if USE_SUPPLY:\n  history = model.fit(\n      train_supply_dataset,\n      #validation_data=val_dataset,\n      epochs=N_EPOCHS ,\n      verbose = 2,\n      callbacks=[\n          save_model_callback(),\n          lr_callback,\n          WeightDecayCallback(),\n          val_lev_callback(),\n          model_checkpoint_callback\n      ]\n  )\n\n  model.save_weights(f\"model_epoch_{N_EPOCHS}_supply.h5\")","metadata":{"id":"ehIq76LDt9ZM","execution":{"iopub.status.busy":"2023-08-27T04:48:14.182269Z","iopub.execute_input":"2023-08-27T04:48:14.182702Z","iopub.status.idle":"2023-08-27T04:48:14.188573Z","shell.execute_reply.started":"2023-08-27T04:48:14.182668Z","shell.execute_reply":"2023-08-27T04:48:14.187604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training on Main dataset","metadata":{"id":"zF8Pq6jtt1KP"}},{"cell_type":"code","source":"if USE_VAL:\n  # Load the pre-train weight of supplyment set here \n  model.load_weights('/content/model_epoch_80_supply.h5')\n\n  history = model.fit(\n      train_dataset,\n      validation_data=val_dataset,\n      epochs=N_EPOCHS ,\n      verbose = 2,\n      callbacks=[\n          save_model_callback(),\n          lr_callback,\n          WeightDecayCallback(),\n          val_lev_callback(),\n          model_checkpoint_callback\n      ]\n  )","metadata":{"id":"b7874611-db8b-4658-9efc-632eab0ba997","outputId":"cacb3b94-7246-4d07-b3a4-4295fe901be1","execution":{"iopub.status.busy":"2023-08-27T04:48:15.27778Z","iopub.execute_input":"2023-08-27T04:48:15.278542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights(f\"model_epoch_{N_EPOCHS}.h5\")","metadata":{"id":"JXdk7iEImrci"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checkpoint for extra epoch and continue training","metadata":{"id":"kd5U4zXK7Jw_"}},{"cell_type":"code","source":"# model.load_weights(\"model_checkpoint.h5\")\n# checkpoint = ModelCheckpoint('model_checkpoint.h5', monitor='loss', verbose=1, save_best_only=True, mode='min')\n\n# optimizer = tfa.optimizers.RectifiedAdam(sma_threshold=4)\n# optimizer = tfa.optimizers.Lookahead(optimizer, sync_period=5)\n# model.compile(optimizer=optimizer, loss=CTCLoss)\n\n# history = model.fit(\n#     train_dataset,\n#     validation_data=val_dataset,\n#     epochs=20,  # Number of extra epochs\n#     verbose=2,\n#     callbacks=[\n#         save_model_callback(),\n#         lr_callback,\n#         WeightDecayCallback(),\n#         val_lev_callback(),\n#         checkpoint  # Updated checkpoint callback\n#     ]\n# )","metadata":{"id":"bdff4f93-31e4-4cd6-93af-098b494661b6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"Agm8u6eb6mYz"},"execution_count":null,"outputs":[]}]}