{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":52950,"databundleVersionId":5973250,"sourceType":"competition"},{"sourceId":8595402,"sourceType":"datasetVersion","datasetId":5142022}],"dockerImageVersionId":30717,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install mediapipe","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-04T17:23:27.028824Z","iopub.execute_input":"2024-06-04T17:23:27.029771Z","iopub.status.idle":"2024-06-04T17:23:43.378309Z","shell.execute_reply.started":"2024-06-04T17:23:27.029724Z","shell.execute_reply":"2024-06-04T17:23:43.377309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip uninstall tensorflow --yes\n!pip uninstall keras --yes\n!pip install keras==2.15 tensorflow==2.15","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:23:43.380474Z","iopub.execute_input":"2024-06-04T17:23:43.380868Z","iopub.status.idle":"2024-06-04T17:24:55.995472Z","shell.execute_reply.started":"2024-06-04T17:23:43.380837Z","shell.execute_reply":"2024-06-04T17:24:55.994573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport json\nimport math\nimport pickle\nimport mediapipe as mp\nfrom mediapipe.framework.formats import landmark_pb2\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport matplotlib\nfrom matplotlib import animation, rc\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:24:55.996961Z","iopub.execute_input":"2024-06-04T17:24:55.997341Z","iopub.status.idle":"2024-06-04T17:25:01.935564Z","shell.execute_reply.started":"2024-06-04T17:24:55.997306Z","shell.execute_reply":"2024-06-04T17:25:01.934599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\nprint(f\"Full train dataset shape is {table.shape}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(table)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_id, file_id, phrase = table.iloc[0][['sequence_id', 'file_id', 'phrase']]\nsample_sequence_df = pq.read_table(f\"/kaggle/input/asl-fingerspelling/train_landmarks/{str(file_id)}.parquet\",\n    filters=[[('sequence_id', '=', sequence_id)],]).to_pandas()\nprint(\"Full sequence dataset shape is {}\".format(sample_sequence_df.shape))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(sample_sequence_df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mediapipe.framework.formats import landmark_pb2\n\nmp_pose = mp.solutions.pose\nmp_hands = mp.solutions.hands\nmp_face_mesh = mp.solutions.face_mesh\nmp_drawing = mp.solutions.drawing_utils\nmp_drawing_styles = mp.solutions.drawing_styles\n\ndef get_pose_hands_face(seq_df):\n    images = []\n    all_pose_landmarks = []\n    all_hand_landmarks = []\n    all_face_landmarks = []\n\n    for seq_idx in range(len(seq_df)):\n        # Выделяем точки позы\n        pose_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for i in range(33):\n            x = seq_df.iloc[seq_idx][f'x_pose_{i}']\n            y = seq_df.iloc[seq_idx][f'y_pose_{i}']\n            z = seq_df.iloc[seq_idx][f'z_pose_{i}']\n            pose_landmarks.landmark.add(x=x, y=y, z=z)\n\n        # Выделяем точки правой руки\n        right_hand_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for i in range(21):\n            x = seq_df.iloc[seq_idx][f'x_right_hand_{i}']\n            y = seq_df.iloc[seq_idx][f'y_right_hand_{i}']\n            z = seq_df.iloc[seq_idx][f'z_right_hand_{i}']\n            right_hand_landmarks.landmark.add(x=x, y=y, z=z)\n\n        # Выделяем точки левой руки\n        left_hand_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for i in range(21):\n            x = seq_df.iloc[seq_idx][f'x_left_hand_{i}']\n            y = seq_df.iloc[seq_idx][f'y_left_hand_{i}']\n            z = seq_df.iloc[seq_idx][f'z_left_hand_{i}']\n            left_hand_landmarks.landmark.add(x=x, y=y, z=z)\n\n        # Выделяем точки лица\n        face_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for i in range(468):\n            x = seq_df.iloc[seq_idx][f'x_face_{i}']\n            y = seq_df.iloc[seq_idx][f'y_face_{i}']\n            z = seq_df.iloc[seq_idx][f'z_face_{i}']\n            face_landmarks.landmark.add(x=x, y=y, z=z)\n\n        image = np.zeros((600, 600, 3))\n\n        # Draw pose\n        mp_drawing.draw_landmarks(\n            image,\n            pose_landmarks,\n            mp_pose.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())\n\n        # Draw right hand\n        mp_drawing.draw_landmarks(\n            image,\n            right_hand_landmarks,\n            mp_hands.HAND_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n\n        # Draw left hand\n        mp_drawing.draw_landmarks(\n            image,\n            left_hand_landmarks,\n            mp_hands.HAND_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n\n        # Draw face\n        mp_drawing.draw_landmarks(\n            image,\n            face_landmarks,\n            mp_face_mesh.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style())\n\n        images.append(image.astype(np.uint8))\n        all_pose_landmarks.append(pose_landmarks)\n        all_hand_landmarks.append((right_hand_landmarks, left_hand_landmarks))\n        all_face_landmarks.append(face_landmarks)\n\n    return images, all_pose_landmarks, all_hand_landmarks, all_face_landmarks\n\ndef create_animation(images):\n    fig = plt.figure(figsize=(6, 9))\n    ax = plt.Axes(fig, [0., 0., 1., 1.])\n    ax.set_axis_off()\n    fig.add_axes(ax)\n    im = ax.imshow(images[0])\n\n    def animate_func(i):\n        im.set_data(images[i])\n        return [im]\n\n    return animation.FuncAnimation(fig, animate_func, frames=len(images), interval=1000/10)\n\n# Get images and landmarks for pose, hands and face\nimages, pose_landmarks, hand_landmarks, face_landmarks = get_pose_hands_face(sample_sequence_df)\n\n# Create animation from images\ncreate_animation(np.array(images))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_subplots(images):\n    num_rows = 5\n    num_cols = 5\n    num_frames = len(images)\n    frame_interval = 1\n\n    fig, axs = plt.subplots(num_rows, num_cols, figsize=(12, 9))\n    for i in range(num_rows):\n        for j in range(num_cols):\n            ax = axs[i, j]\n            ax.set_axis_off()\n            start_frame = i * num_cols * frame_interval + j * frame_interval\n            end_frame = start_frame + frame_interval\n            if end_frame > num_frames:\n                end_frame = num_frames\n            for k in range(start_frame, end_frame):\n                ax.imshow(images[k])\n            ax.set_title(f'Кадр {start_frame}')\n\n    plt.tight_layout()\n    plt.show()\n    fig.savefig('/kaggle/working/image2.png')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_subplots(np.array(images))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\n#Добавим символ конца фразы '^'\npad_token = '^'\npad_token_idx = 59\n\nchar_to_num[pad_token] = pad_token_idx\nprint(char_to_num)\nnum_to_char = {j:i for i,j in char_to_num.items()}\ndf = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n\n#Добавим индексы лица(конкретно губ), а также правой и левой рук человека соответсвенно с документацией Mediapipe\nLIP = [\n    61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\n#Создаём списки с именами координат\nX = [f'x_right_hand_{i}' for i in range(21)]  +[f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE] + [f'x_face_{i}' for i in LIP] #+ \nY = [f'y_right_hand_{i}' for i in range(21)]  +[f'y_left_hand_{i}' for i in range(21)]+ [f'y_pose_{i}' for i in POSE] + [f'y_face_{i}' for i in LIP] #+\nZ = [f'z_right_hand_{i}' for i in range(21)]  + [f'z_left_hand_{i}' for i in range(21)]+ [f'z_pose_{i}' for i in POSE] + [f'z_face_{i}' for i in LIP] #+ \n\n#Получим списки из порядковых номеров необходимых координат\nSEL_COLS = X + Y + Z\nFRAME_LEN = 128 + 48\nMAX_PHRASE_LENGTH = 64\n\nLIP_IDX_X   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"x\" in col]\nRHAND_IDX_X = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"x\" in col]\nLHAND_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"x\" in col]\nRPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE and \"x\" in col]\nLPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE and \"x\" in col]\n\nLIP_IDX_Y   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"y\" in col]\nRHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"y\" in col]\nLHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"y\" in col]\nRPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE and \"y\" in col]\nLPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE and \"y\" in col]\n\nLIP_IDX_Z   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"z\" in col]\nRHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"z\" in col]\nLHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"z\" in col]\nRPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE and \"z\" in col]\nLPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE and \"z\" in col]\n\n#загрузим подгтовленные записи TFRec для матожидания и среднеквадратического отклонения для меток\nRHM = np.load(\"/kaggle/input/tfrecords-data/mean_std/rh_mean.npy\")\nLHM = np.load(\"/kaggle/input/tfrecords-data/mean_std/lh_mean.npy\")\nRPM = np.load(\"/kaggle/input/tfrecords-data/mean_std/rp_mean.npy\")\nLPM = np.load(\"/kaggle/input/tfrecords-data/mean_std/lp_mean.npy\")\nLIPM = np.load(\"/kaggle/input/tfrecords-data/mean_std/lip_mean.npy\")\n\nRHS = np.load(\"/kaggle/input/tfrecords-data//mean_std/rh_std.npy\")\nLHS = np.load(\"/kaggle/input/tfrecords-data/mean_std/lh_std.npy\")\nRPS = np.load(\"/kaggle/input/tfrecords-data/mean_std/rp_std.npy\")\nLPS = np.load(\"/kaggle/input/tfrecords-data/mean_std/lp_std.npy\")\nLIPS = np.load(\"/kaggle/input/tfrecords-data/mean_std/lip_std.npy\")","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:25:01.938589Z","iopub.execute_input":"2024-06-04T17:25:01.939311Z","iopub.status.idle":"2024-06-04T17:25:02.200239Z","shell.execute_reply.started":"2024-06-04T17:25:01.939276Z","shell.execute_reply":"2024-06-04T17:25:02.199389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=SEL_COLS)\n\nfile_id = df.file_id.iloc[0]\ninpdir = \"/kaggle/input/asl-fingerspelling/train_landmarks\"\npqfile = f\"{inpdir}/{file_id}.parquet\"\nseq_refs = df.loc[df.file_id == file_id]\nseqs = load_relevant_data_subset(pqfile)\n\nseq_id = seq_refs.sequence_id.iloc[0]\nframes = seqs.iloc[seqs.index == seq_id]\nphrase = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:25:02.201442Z","iopub.execute_input":"2024-06-04T17:25:02.20181Z","iopub.status.idle":"2024-06-04T17:25:06.063827Z","shell.execute_reply.started":"2024-06-04T17:25:02.201775Z","shell.execute_reply":"2024-06-04T17:25:06.062709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function()\ndef resize_pad(x):\n    if tf.shape(x)[0] < FRAME_LEN:\n        x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]), constant_values=float(\"NaN\"))\n    else:\n        x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n    return x\n\ndef pre_process_get_mask(x):\n    lip_x = tf.gather(x, LIP_IDX_X, axis=1)\n    lip_y = tf.gather(x, LIP_IDX_Y, axis=1)\n    lip_z = tf.gather(x, LIP_IDX_Z, axis=1)\n\n    rhand_x = tf.gather(x, RHAND_IDX_X, axis=1)\n    rhand_y = tf.gather(x, RHAND_IDX_Y, axis=1)\n    rhand_z = tf.gather(x, RHAND_IDX_Z, axis=1)\n    \n    lhand_x = tf.gather(x, LHAND_IDX_X, axis=1)\n    lhand_y = tf.gather(x, LHAND_IDX_Y, axis=1)\n    lhand_z = tf.gather(x, LHAND_IDX_Z, axis=1)\n\n    rpose_x = tf.gather(x, RPOSE_IDX_X, axis=1)\n    rpose_y = tf.gather(x, RPOSE_IDX_Y, axis=1)\n    rpose_z = tf.gather(x, RPOSE_IDX_Z, axis=1)\n    \n    lpose_x = tf.gather(x, LPOSE_IDX_X, axis=1)\n    lpose_y = tf.gather(x, LPOSE_IDX_Y, axis=1)\n    lpose_z = tf.gather(x, LPOSE_IDX_Z, axis=1)\n    \n    lip   = tf.concat([lip_x[..., tf.newaxis], lip_y[..., tf.newaxis], lip_z[..., tf.newaxis]], axis=-1)\n    rhand = tf.concat([rhand_x[..., tf.newaxis], rhand_y[..., tf.newaxis], rhand_z[..., tf.newaxis]], axis=-1)\n    lhand = tf.concat([lhand_x[..., tf.newaxis], lhand_y[..., tf.newaxis], lhand_z[..., tf.newaxis]], axis=-1)\n    rpose = tf.concat([rpose_x[..., tf.newaxis], rpose_y[..., tf.newaxis], rpose_z[..., tf.newaxis]], axis=-1)\n    lpose = tf.concat([lpose_x[..., tf.newaxis], lpose_y[..., tf.newaxis], lpose_z[..., tf.newaxis]], axis=-1)\n        \n    # TIME AUGMENTATION\n    if tf.random.uniform(shape=(), minval=0, maxval=1)<0.2:\n        new_width = tf.shape(lip)[1]\n        new_height = tf.random.uniform(\n            shape=(), minval = tf.cast(tf.shape(lip)[0],tf.float32) / 2.0, \n            maxval = tf.cast(tf.shape(lip)[0],tf.float32) * 1.5, dtype=tf.int32)\n        resized_lip = tf.image.resize(lip, (new_height, new_width))\n        resized_rhand = tf.image.resize(rhand, (new_height, new_width))\n        resized_lhand = tf.image.resize(lhand, (new_height, new_width))\n        resized_rpose = tf.image.resize(rpose, (new_height, new_width))\n        resized_lpose = tf.image.resize(lpose, (new_height, new_width))\n        return resized_lip, resized_rhand, resized_lhand, resized_rpose, resized_lpose        \n        \n    return lip, rhand, lhand,  rpose, lpose \n\n@tf.function()\ndef pre_process_shape(lip, rhand,lhand,  rpose, lpose):\n    lip   = (resize_pad(lip) - LIPM) / LIPS\n    rhand = (resize_pad(rhand) - RHM) / RHS\n    lhand = (resize_pad(lhand) - LHM) / LHS\n    rpose = (resize_pad(rpose) - RPM) / RPS\n    lpose = (resize_pad(lpose) - LPM) / LPS\n\n    x = tf.concat([lip, rhand, lhand, rpose, lpose], axis=1)\n    s = tf.shape(x)\n    x = tf.reshape(x, (s[0], s[1]*s[2]))\n    x = tf.where(tf.math.is_nan(x), 0.0, x)\n    return x\n\npre0 = pre_process_get_mask(frames)\npre1 = pre_process_shape(*pre0)\nINPUT_SHAPE = list(pre1.shape)\nprint(INPUT_SHAPE)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:25:06.065733Z","iopub.execute_input":"2024-06-04T17:25:06.066247Z","iopub.status.idle":"2024-06-04T17:25:08.159405Z","shell.execute_reply.started":"2024-06-04T17:25:06.066212Z","shell.execute_reply":"2024-06-04T17:25:08.15831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function(jit_compile=True)\ndef pre_process_data(x):\n    lip_x = tf.gather(x, LIP_IDX_X, axis=1)\n    lip_y = tf.gather(x, LIP_IDX_Y, axis=1)\n    lip_z = tf.gather(x, LIP_IDX_Z, axis=1)\n\n    rhand_x = tf.gather(x, RHAND_IDX_X, axis=1)\n    rhand_y = tf.gather(x, RHAND_IDX_Y, axis=1)\n    rhand_z = tf.gather(x, RHAND_IDX_Z, axis=1)\n    \n    lhand_x = tf.gather(x, LHAND_IDX_X, axis=1)\n    lhand_y = tf.gather(x, LHAND_IDX_Y, axis=1)\n    lhand_z = tf.gather(x, LHAND_IDX_Z, axis=1)\n\n    rpose_x = tf.gather(x, RPOSE_IDX_X, axis=1)\n    rpose_y = tf.gather(x, RPOSE_IDX_Y, axis=1)\n    rpose_z = tf.gather(x, RPOSE_IDX_Z, axis=1)\n    \n    lpose_x = tf.gather(x, LPOSE_IDX_X, axis=1)\n    lpose_y = tf.gather(x, LPOSE_IDX_Y, axis=1)\n    lpose_z = tf.gather(x, LPOSE_IDX_Z, axis=1)\n    \n    lip   = tf.concat([lip_x[..., tf.newaxis], lip_y[..., tf.newaxis], lip_z[..., tf.newaxis]], axis=-1)\n    rhand = tf.concat([rhand_x[..., tf.newaxis], rhand_y[..., tf.newaxis], rhand_z[..., tf.newaxis]], axis=-1)\n    lhand = tf.concat([lhand_x[..., tf.newaxis], lhand_y[..., tf.newaxis], lhand_z[..., tf.newaxis]], axis=-1)\n    rpose = tf.concat([rpose_x[..., tf.newaxis], rpose_y[..., tf.newaxis], rpose_z[..., tf.newaxis]], axis=-1)\n    lpose = tf.concat([lpose_x[..., tf.newaxis], lpose_y[..., tf.newaxis], lpose_z[..., tf.newaxis]], axis=-1)\n                \n    hand = tf.concat([rhand, lhand], axis=1)\n    hand = tf.where(tf.math.is_nan(hand), 0.0, hand)\n    input_tensor = tf.math.not_equal(tf.reduce_sum(hand, axis=[1, 2]), 0.0)\n    alternating_tensor = tf.math.equal( tf.cumsum(tf.ones_like( tf.reduce_sum(hand, axis=[1, 2]) ))%2, 1.0 )\n    mask = tf.math.logical_or(input_tensor, alternating_tensor)\n    \n    lip = lip[mask]\n    rhand = rhand[mask]\n    lhand = lhand[mask]\n    rpose = rpose[mask]\n    lpose = lpose[mask]\n\n    return lip, rhand, lhand,  rpose, lpose","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:25:08.160835Z","iopub.execute_input":"2024-06-04T17:25:08.161257Z","iopub.status.idle":"2024-06-04T17:25:08.181155Z","shell.execute_reply.started":"2024-06-04T17:25:08.161223Z","shell.execute_reply":"2024-06-04T17:25:08.180091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_fn(record_bytes):\n    schema = {\n        \"lip\": tf.io.VarLenFeature(tf.float32),\n        \"rhand\": tf.io.VarLenFeature(tf.float32),\n        \"lhand\": tf.io.VarLenFeature(tf.float32),\n        \"rpose\": tf.io.VarLenFeature(tf.float32),\n        \"lpose\": tf.io.VarLenFeature(tf.float32),\n        \"phrase\": tf.io.VarLenFeature(tf.int64)\n    }\n\n    x = tf.io.parse_single_example(record_bytes, schema)\n\n    lip = tf.reshape(tf.sparse.to_dense(x[\"lip\"]), (-1, 40, 3))\n    rhand = tf.reshape(tf.sparse.to_dense(x[\"rhand\"]), (-1, 21, 3))\n    lhand = tf.reshape(tf.sparse.to_dense(x[\"lhand\"]), (-1, 21, 3))\n    rpose = tf.reshape(tf.sparse.to_dense(x[\"rpose\"]), (-1, 5, 3))\n    lpose = tf.reshape(tf.sparse.to_dense(x[\"lpose\"]), (-1, 5, 3))\n    phrase = tf.sparse.to_dense(x[\"phrase\"])\n\n    return lip, rhand, lhand,  rpose, lpose, phrase \n\ndef pre_process_fn(lip, rhand, lhand, rpose, lpose, phrase): \n    phrase = tf.pad(phrase, [[0, MAX_PHRASE_LENGTH-tf.shape(phrase)[0]]], constant_values=pad_token_idx)\n    return pre_process_shape(lip, rhand, lhand, rpose, lpose), phrase \n    \ntffiles = [f\"/kaggle/input/tfrecords-data/tfds/{file_id}.tfrecord\" for file_id in df.file_id.unique()]\nval_len = 1\ntrain_batch_size = 128\nval_batch_size = 128\n\ntrain_dataset =  tf.data.TFRecordDataset(tffiles[val_len:]).prefetch(tf.data.AUTOTUNE).shuffle(5000).map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE).map(pre_process_fn, num_parallel_calls=tf.data.AUTOTUNE).batch(train_batch_size).prefetch(tf.data.AUTOTUNE)\nval_dataset =  tf.data.TFRecordDataset(tffiles[:val_len]).prefetch(tf.data.AUTOTUNE).map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE).map(pre_process_fn, num_parallel_calls=tf.data.AUTOTUNE).batch(val_batch_size).prefetch(tf.data.AUTOTUNE)\n\nbatch = next(iter(val_dataset))\nbatch[0].shape, batch[1].shape","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:25:08.182418Z","iopub.execute_input":"2024-06-04T17:25:08.18279Z","iopub.status.idle":"2024-06-04T17:25:08.98008Z","shell.execute_reply.started":"2024-06-04T17:25:08.18276Z","shell.execute_reply":"2024-06-04T17:25:08.979092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# СОЗДАНИЕ КЛАССОВ ДЛЯ МОДЕЛИ","metadata":{}},{"cell_type":"code","source":"class ECA(tf.keras.layers.Layer):\n    def __init__(self, kernel_size=5, **kwargs):\n        super().__init__(**kwargs)\n        self.supports_masking = True\n        self.kernel_size = kernel_size\n        self.conv = tf.keras.layers.Conv1D(1, kernel_size=kernel_size, strides=1, padding=\"same\", use_bias=False)\n\n    def call(self, inputs, mask=None):\n        nn = tf.keras.layers.GlobalAveragePooling1D()(inputs, mask=mask)\n        nn = tf.expand_dims(nn, -1)\n        nn = self.conv(nn)\n        nn = tf.squeeze(nn, -1)\n        nn = tf.nn.sigmoid(nn)\n        nn = nn[:,None,:]\n        return inputs * nn\n\nclass CausalDWConv1D(tf.keras.layers.Layer):\n    def __init__(self, \n        kernel_size=17,\n        dilation_rate=1,\n        use_bias=False,\n        depthwise_initializer='glorot_uniform',\n        name='', **kwargs):\n        super().__init__(name=name,**kwargs)\n        self.causal_pad = tf.keras.layers.ZeroPadding1D((dilation_rate*(kernel_size-1),0),name=name + '_pad')\n        self.dw_conv = tf.keras.layers.DepthwiseConv1D(\n                            kernel_size,\n                            strides=1,\n                            dilation_rate=dilation_rate,\n                            padding='valid',\n                            use_bias=use_bias,\n                            depthwise_initializer=depthwise_initializer,\n                            name=name + '_dwconv')\n        self.supports_masking = True\n        \n    def call(self, inputs):\n        x = self.causal_pad(inputs)\n        x = self.dw_conv(x)\n        return x\n\ndef Conv1DBlock(channel_size,\n          kernel_size,\n          dilation_rate=1,\n          drop_rate=0.0,\n          expand_ratio=2,\n          se_ratio=0.25,\n          activation='swish',\n          name=None):\n    '''\n    efficient conv1d block, @hoyso48\n    '''\n    if name is None:\n        name = str(tf.keras.backend.get_uid(\"mbblock\"))\n    # Expansion phase\n    def apply(inputs):\n        channels_in = tf.keras.backend.int_shape(inputs)[-1]\n        channels_expand = channels_in * expand_ratio\n\n        skip = inputs\n\n        x = tf.keras.layers.Dense(\n            channels_expand,\n            use_bias=True,\n            activation=activation,\n            name=name + '_expand_conv')(inputs)\n\n        # Depthwise Convolution\n        x = CausalDWConv1D(kernel_size,\n            dilation_rate=dilation_rate,\n            use_bias=False,\n            name=name + '_dwconv')(x)\n\n        x = tf.keras.layers.BatchNormalization(momentum=0.95, name=name + '_bn')(x)\n\n        x  = ECA()(x)\n\n        x = tf.keras.layers.Dense(\n            channel_size,\n            use_bias=True,\n            name=name + '_project_conv')(x)\n\n        if drop_rate > 0:\n            x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1), name=name + '_drop')(x)\n\n        if (channels_in == channel_size):\n            x = tf.keras.layers.add([x, skip], name=name + '_add')\n        return x\n\n    return apply\n\nclass MultiHeadSelfAttention(tf.keras.layers.Layer):\n    def __init__(self, dim=256, num_heads=4, dropout=0, **kwargs):\n        super().__init__(**kwargs)\n        self.dim = dim\n        self.scale = self.dim ** -0.5\n        self.num_heads = num_heads\n        self.qkv = tf.keras.layers.Dense(3 * dim, use_bias=False)\n        self.drop1 = tf.keras.layers.Dropout(dropout)\n        self.proj = tf.keras.layers.Dense(dim, use_bias=False)\n        self.supports_masking = True\n\n    def call(self, inputs, mask=None):\n        qkv = self.qkv(inputs)\n        qkv = tf.keras.layers.Permute((2, 1, 3))(tf.keras.layers.Reshape((-1, self.num_heads, self.dim * 3 // self.num_heads))(qkv))\n        q, k, v = tf.split(qkv, [self.dim // self.num_heads] * 3, axis=-1)\n\n        attn = tf.matmul(q, k, transpose_b=True) * self.scale\n\n        if mask is not None:\n            mask = mask[:, None, None, :]\n\n        attn = tf.keras.layers.Softmax(axis=-1)(attn, mask=mask)\n        attn = self.drop1(attn)\n\n        x = attn @ v\n        x = tf.keras.layers.Reshape((-1, self.dim))(tf.keras.layers.Permute((2, 1, 3))(x))\n        x = self.proj(x)\n        return x\n\n\ndef TransformerBlock(dim=256, num_heads=6, expand=4, attn_dropout=0.2, drop_rate=0.2, activation='swish'):\n    def apply(inputs):\n        x = inputs\n        x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n        x = MultiHeadSelfAttention(dim=dim,num_heads=num_heads,dropout=attn_dropout)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([inputs, x])\n        attn_out = x\n\n        x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n        x = tf.keras.layers.Dense(dim*expand, use_bias=False, activation=activation)(x)\n        x = tf.keras.layers.Dense(dim, use_bias=False)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([attn_out, x])\n        return x\n    return apply\n\ndef positional_encoding(maxlen, num_hid):\n        depth = num_hid/2\n        positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n        depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n        angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n        angle_rads = tf.linalg.matmul(positions, angle_rates)\n        pos_encoding = tf.concat(\n          [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n          axis=-1)\n        return pos_encoding","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:25:08.981445Z","iopub.execute_input":"2024-06-04T17:25:08.981772Z","iopub.status.idle":"2024-06-04T17:25:09.038506Z","shell.execute_reply.started":"2024-06-04T17:25:08.981748Z","shell.execute_reply":"2024-06-04T17:25:09.037559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CTC Блок","metadata":{}},{"cell_type":"code","source":"# Блок CTC\ndef CTCLoss(labels, logits):\n    label_length = tf.reduce_sum(tf.cast(labels != pad_token_idx, tf.int32), axis=-1)\n    logit_length = tf.ones(tf.shape(logits)[0], dtype=tf.int32) * tf.shape(logits)[1]\n    loss = tf.nn.ctc_loss(\n            labels=labels,\n            logits=logits,\n            label_length=label_length,\n            logit_length=logit_length,\n            blank_index=pad_token_idx,\n            logits_time_major=False\n        )\n    loss = tf.reduce_mean(loss)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:25:09.041696Z","iopub.execute_input":"2024-06-04T17:25:09.042099Z","iopub.status.idle":"2024-06-04T17:25:09.04894Z","shell.execute_reply.started":"2024-06-04T17:25:09.042032Z","shell.execute_reply":"2024-06-04T17:25:09.047771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # Создание классов для оптимизатора\n","metadata":{}},{"cell_type":"markdown","source":"# Lookahead","metadata":{}},{"cell_type":"code","source":"from typing import Union, Callable, List, Dict\n\nKerasLegacyOptimizer = tf.keras.optimizers.legacy.Optimizer\nFloatTensorLike = Union[tf.Tensor, float, np.float16, np.float32, np.float64]\n\n@tf.keras.utils.register_keras_serializable(package=\"Addons\")\nclass Lookahead(KerasLegacyOptimizer):\n\n    def __init__(\n        self,\n        optimizer: Union[tf.keras.optimizers.Optimizer, tf.keras.optimizers.legacy.Optimizer, str],\n        sync_period: int = 6,\n        slow_step_size: FloatTensorLike = 0.5,\n        name: str = \"Lookahead\",\n        **kwargs,\n    ):\n\n        super().__init__(name, **kwargs)\n\n        if isinstance(optimizer, str):\n            if (\n                hasattr(tf.keras.optimizers, \"legacy\")\n                and KerasLegacyOptimizer == tf.keras.optimizers.legacy.Optimizer\n            ):\n                optimizer = tf.keras.optimizers.get(\n                    optimizer, use_legacy_optimizer=True\n                )\n            else:\n                optimizer = tf.keras.optimizers.get(optimizer)\n        if not isinstance(optimizer, KerasLegacyOptimizer):\n            raise TypeError(\n                \"optimizer is not an object of tf.keras.optimizers.legacy.Optimizer \"\n            )\n\n        self._optimizer = optimizer\n        self._set_hyper(\"sync_period\", sync_period)\n        self._set_hyper(\"slow_step_size\", slow_step_size)\n        self._initialized = False\n        self._track_trackable(self._optimizer, \"lh_base_optimizer\")\n\n    def _create_slots(self, var_list):\n        self._optimizer._create_slots(\n            var_list=var_list\n        )  # pylint: disable=protected-access\n        for var in var_list:\n            self.add_slot(var, \"slow\", initializer=var)\n\n    def _create_hypers(self):\n        self._optimizer._create_hypers()  # pylint: disable=protected-access\n\n    def _prepare(self, var_list):\n        return self._optimizer._prepare(\n            var_list=var_list\n        )  # pylint: disable=protected-access\n\n    def apply_gradients(self, grads_and_vars, name=None, **kwargs):\n        self._optimizer._iterations = (\n            self.iterations\n        )  # pylint: disable=protected-access\n        return super().apply_gradients(grads_and_vars, name, **kwargs)\n\n    def _look_ahead_op(self, var):\n        var_dtype = var.dtype.base_dtype\n        slow_var = self.get_slot(var, \"slow\")\n        local_step = tf.cast(self.iterations + 1, tf.dtypes.int64)\n        sync_period = self._get_hyper(\"sync_period\", tf.dtypes.int64)\n        slow_step_size = self._get_hyper(\"slow_step_size\", var_dtype)\n        step_back = slow_var + slow_step_size * (var - slow_var)\n        sync_cond = tf.equal(\n            tf.math.floordiv(local_step, sync_period) * sync_period, local_step\n        )\n        with tf.control_dependencies([step_back]):\n            slow_update = slow_var.assign(\n                tf.where(sync_cond, step_back, slow_var),\n                use_locking=self._use_locking,\n            )\n            var_update = var.assign(\n                tf.where(sync_cond, step_back, var),\n                use_locking=self._use_locking,\n            )\n        return tf.group(slow_update, var_update)\n\n    @property\n    def weights(self):\n        return self._weights + self._optimizer.weights\n\n    def _resource_apply_dense(self, grad, var):\n        train_op = self._optimizer._resource_apply_dense(\n            grad, var\n        )  # pylint: disable=protected-access\n        with tf.control_dependencies([train_op]):\n            look_ahead_op = self._look_ahead_op(var)\n        return tf.group(train_op, look_ahead_op)\n\n    def _resource_apply_sparse(self, grad, var, indices):\n        train_op = (\n            self._optimizer._resource_apply_sparse(  # pylint: disable=protected-access\n                grad, var, indices\n            )\n        )\n        with tf.control_dependencies([train_op]):\n            look_ahead_op = self._look_ahead_op(var)\n        return tf.group(train_op, look_ahead_op)\n\n    def get_config(self):\n        config = {\n            \"optimizer\": tf.keras.optimizers.serialize(self._optimizer),\n            \"sync_period\": self._serialize_hyperparameter(\"sync_period\"),\n            \"slow_step_size\": self._serialize_hyperparameter(\"slow_step_size\"),\n        }\n        base_config = super().get_config()\n        return {**base_config, **config}\n\n    @property\n    def learning_rate(self):\n        return self._optimizer._get_hyper(\"learning_rate\")\n\n    @learning_rate.setter\n    def learning_rate(self, learning_rate):\n        self._optimizer._set_hyper(\"learning_rate\", learning_rate)\n\n    @property\n    def lr(self):\n        return self.learning_rate\n\n    @lr.setter\n    def lr(self, lr):\n        self.learning_rate = lr\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        optimizer = tf.keras.optimizers.deserialize(\n            config.pop(\"optimizer\"), custom_objects=custom_objects\n        )\n        return cls(optimizer, **config)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:25:09.050965Z","iopub.execute_input":"2024-06-04T17:25:09.051332Z","iopub.status.idle":"2024-06-04T17:25:09.07885Z","shell.execute_reply.started":"2024-06-04T17:25:09.051302Z","shell.execute_reply":"2024-06-04T17:25:09.07784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RectifiedAdam","metadata":{}},{"cell_type":"code","source":"from typing import Dict\nclass RectifiedAdam(KerasLegacyOptimizer):\n\n    def __init__(\n        self,\n        learning_rate: Union[FloatTensorLike, Callable, Dict] = 0.001,\n        beta_1: FloatTensorLike = 0.9,\n        beta_2: FloatTensorLike = 0.999,\n        epsilon: FloatTensorLike = 1e-7,\n        weight_decay: Union[FloatTensorLike, Callable, Dict] = 0.0,\n        amsgrad: bool = False,\n        sma_threshold: FloatTensorLike = 5.0,\n        total_steps: int = 0,\n        warmup_proportion: FloatTensorLike = 0.1,\n        min_lr: FloatTensorLike = 0.0,\n        name: str = \"RectifiedAdam\",\n        **kwargs,\n    ):\n\n        super().__init__(name, **kwargs)\n        print(learning_rate)\n        if isinstance(learning_rate, Dict):\n            learning_rate = tf.keras.optimizers.schedules.deserialize(learning_rate)\n\n        if isinstance(weight_decay, Dict):\n            weight_decay = tf.keras.optimizers.schedules.deserialize(weight_decay)\n\n        self._set_hyper(\"learning_rate\", kwargs.get(\"lr\", learning_rate))\n        self._set_hyper(\"beta_1\", beta_1)\n        self._set_hyper(\"beta_2\", beta_2)\n        self._set_hyper(\"decay\", self._initial_decay)\n        self._set_hyper(\"weight_decay\", weight_decay)\n        self._set_hyper(\"sma_threshold\", sma_threshold)\n        self._set_hyper(\"total_steps\", float(total_steps))\n        self._set_hyper(\"warmup_proportion\", warmup_proportion)\n        self._set_hyper(\"min_lr\", min_lr)\n        self.epsilon = epsilon or tf.keras.backend.epsilon()\n        self.amsgrad = amsgrad\n        self._has_weight_decay = weight_decay != 0.0\n        self._initial_total_steps = total_steps\n\n    def _create_slots(self, var_list):\n        for var in var_list:\n            self.add_slot(var, \"m\")\n        for var in var_list:\n            self.add_slot(var, \"v\")\n        if self.amsgrad:\n            for var in var_list:\n                self.add_slot(var, \"vhat\")\n\n    def set_weights(self, weights):\n        params = self.weights\n        num_vars = int((len(params) - 1) / 2)\n        if len(weights) == 3 * num_vars + 1:\n            weights = weights[: len(params)]\n        super().set_weights(weights)\n\n    def _decayed_wd(self, var_dtype):\n        wd_t = self._get_hyper(\"weight_decay\", var_dtype)\n        if isinstance(wd_t, tf.keras.optimizers.schedules.LearningRateSchedule):\n            wd_t = tf.cast(wd_t(self.iterations), var_dtype)\n        return wd_t\n\n    def _prepare_local(self, var_device, var_dtype, apply_state):\n        super()._prepare_local(var_device, var_dtype, apply_state)\n        lr_t = self._decayed_lr(var_dtype)\n        wd_t = self._decayed_wd(var_dtype)\n        beta_1_t = self._get_hyper(\"beta_1\", var_dtype)\n        beta_2_t = self._get_hyper(\"beta_2\", var_dtype)\n        local_step = tf.cast(self.iterations + 1, var_dtype)\n        beta_1_power = tf.pow(beta_1_t, local_step)\n        beta_2_power = tf.pow(beta_2_t, local_step)\n        one_minus_beta_1_t = 1.0 - beta_1_t\n        recip_one_minus_beta_1_power = 1.0 / (1.0 - beta_1_power)\n        one_minus_beta_2_t = 1.0 - beta_2_t\n        recip_one_minus_beta_2_power = 1.0 / (1.0 - beta_2_power)\n        sma_inf = 2.0 / one_minus_beta_2_t - 1.0\n        sma_t = sma_inf - 2.0 * local_step * beta_2_power * recip_one_minus_beta_2_power\n        r_t = tf.sqrt(\n            (sma_t - 4.0)\n            / (sma_inf - 4.0)\n            * (sma_t - 2.0)\n            / (sma_inf - 2.0)\n            * sma_inf\n            / sma_t\n        )\n        sma_threshold = self._get_hyper(\"sma_threshold\", var_dtype)\n        sma_t_ge_sma_threshold = sma_t >= sma_threshold\n        if self._initial_total_steps > 0:\n            total_steps = self._get_hyper(\"total_steps\", var_dtype)\n            warmup_steps = total_steps * self._get_hyper(\"warmup_proportion\", var_dtype)\n            min_lr = self._get_hyper(\"min_lr\", var_dtype)\n            decay_steps = tf.maximum(total_steps - warmup_steps, 1)\n            decay_rate = (min_lr - lr_t) / decay_steps\n            lr_t = tf.where(\n                local_step <= warmup_steps,\n                lr_t * (local_step / warmup_steps),\n                lr_t + decay_rate * tf.minimum(local_step - warmup_steps, decay_steps),\n            )\n        apply_state[(var_device, var_dtype)].update(\n            dict(\n                lr_t=lr_t,\n                wd_t=wd_t,\n                beta_1_t=beta_1_t,\n                beta_2_t=beta_2_t,\n                epsilon_t=tf.convert_to_tensor(self.epsilon, var_dtype),\n                local_step=local_step,\n                beta_1_power=beta_1_power,\n                beta_2_power=beta_2_power,\n                sma_inf=sma_inf,\n                sma_t=sma_t,\n                one_minus_beta_1_t=one_minus_beta_1_t,\n                recip_one_minus_beta_1_power=recip_one_minus_beta_1_power,\n                one_minus_beta_2_t=one_minus_beta_2_t,\n                recip_one_minus_beta_2_power=recip_one_minus_beta_2_power,\n                r_t=r_t,\n                sma_t_ge_sma_threshold=sma_t_ge_sma_threshold,\n            )\n        )\n\n    def _resource_apply_dense(self, grad, var, apply_state=None):\n        var_device, var_dtype = var.device, var.dtype.base_dtype\n        coef = (apply_state or {}).get(\n            (var_device, var_dtype)\n        ) or self._fallback_apply_state(var_device, var_dtype)\n        m = self.get_slot(var, \"m\")\n        v = self.get_slot(var, \"v\")\n\n        m_t = m.assign(\n            coef[\"beta_1_t\"] * m + coef[\"one_minus_beta_1_t\"] * grad,\n            use_locking=self._use_locking,\n        )\n        m_corr_t = m_t * coef[\"recip_one_minus_beta_1_power\"]\n\n        v_t = v.assign(\n            coef[\"beta_2_t\"] * v + coef[\"one_minus_beta_2_t\"] * tf.square(grad),\n            use_locking=self._use_locking,\n        )\n        if self.amsgrad:\n            vhat = self.get_slot(var, \"vhat\")\n            vhat_t = vhat.assign(tf.maximum(vhat, v_t), use_locking=self._use_locking)\n            v_corr_t = tf.sqrt(vhat_t * coef[\"recip_one_minus_beta_2_power\"])\n        else:\n            vhat_t = None\n            v_corr_t = tf.sqrt(v_t * coef[\"recip_one_minus_beta_2_power\"])\n\n        var_t = tf.where(\n            coef[\"sma_t_ge_sma_threshold\"],\n            coef[\"r_t\"] * m_corr_t / (v_corr_t + coef[\"epsilon_t\"]),\n            m_corr_t,\n        )\n\n        if self._has_weight_decay:\n            var_t += coef[\"wd_t\"] * var\n\n        var_update = var.assign_sub(coef[\"lr_t\"] * var_t, use_locking=self._use_locking)\n\n        updates = [var_update, m_t, v_t]\n        if self.amsgrad:\n            updates.append(vhat_t)\n        return tf.group(*updates)\n\n    def _resource_apply_sparse(self, grad, var, indices, apply_state=None):\n        var_device, var_dtype = var.device, var.dtype.base_dtype\n        coef = (apply_state or {}).get(\n            (var_device, var_dtype)\n        ) or self._fallback_apply_state(var_device, var_dtype)\n\n        m = self.get_slot(var, \"m\")\n        m_scaled_g_values = grad * coef[\"one_minus_beta_1_t\"]\n        m_t = m.assign(m * coef[\"beta_1_t\"], use_locking=self._use_locking)\n        with tf.control_dependencies([m_t]):\n            m_t = self._resource_scatter_add(m, indices, m_scaled_g_values)\n        m_corr_t = m_t * coef[\"recip_one_minus_beta_1_power\"]\n\n        v = self.get_slot(var, \"v\")\n        v_scaled_g_values = (grad * grad) * coef[\"one_minus_beta_2_t\"]\n        v_t = v.assign(v * coef[\"beta_2_t\"], use_locking=self._use_locking)\n        with tf.control_dependencies([v_t]):\n            v_t = self._resource_scatter_add(v, indices, v_scaled_g_values)\n\n        if self.amsgrad:\n            vhat = self.get_slot(var, \"vhat\")\n            vhat_t = vhat.assign(tf.maximum(vhat, v_t), use_locking=self._use_locking)\n            v_corr_t = tf.sqrt(vhat_t * coef[\"recip_one_minus_beta_2_power\"])\n        else:\n            vhat_t = None\n            v_corr_t = tf.sqrt(v_t * coef[\"recip_one_minus_beta_2_power\"])\n\n        var_t = tf.where(\n            coef[\"sma_t_ge_sma_threshold\"],\n            coef[\"r_t\"] * m_corr_t / (v_corr_t + coef[\"epsilon_t\"]),\n            m_corr_t,\n        )\n\n        if self._has_weight_decay:\n            var_t += coef[\"wd_t\"] * var\n\n        with tf.control_dependencies([var_t]):\n            var_update = self._resource_scatter_add(\n                var, indices, tf.gather(-coef[\"lr_t\"] * var_t, indices)\n            )\n\n        updates = [var_update, m_t, v_t]\n        if self.amsgrad:\n            updates.append(vhat_t)\n        return tf.group(*updates)\n\n    def get_config(self):\n        config = super().get_config()\n        config.update(\n            {\n                \"learning_rate\": self._serialize_hyperparameter(\"learning_rate\"),\n                \"beta_1\": self._serialize_hyperparameter(\"beta_1\"),\n                \"beta_2\": self._serialize_hyperparameter(\"beta_2\"),\n                \"decay\": self._serialize_hyperparameter(\"decay\"),\n                \"weight_decay\": self._serialize_hyperparameter(\"weight_decay\"),\n                \"sma_threshold\": self._serialize_hyperparameter(\"sma_threshold\"),\n                \"epsilon\": self.epsilon,\n                \"amsgrad\": self.amsgrad,\n                \"total_steps\": int(self._serialize_hyperparameter(\"total_steps\")),\n                \"warmup_proportion\": self._serialize_hyperparameter(\n                    \"warmup_proportion\"\n                ),\n                \"min_lr\": self._serialize_hyperparameter(\"min_lr\"),\n            }\n        )\n        return config","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:25:09.080471Z","iopub.execute_input":"2024-06-04T17:25:09.081191Z","iopub.status.idle":"2024-06-04T17:25:09.122519Z","shell.execute_reply.started":"2024-06-04T17:25:09.081163Z","shell.execute_reply":"2024-06-04T17:25:09.121428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(dim = 384):\n    \n    #Input\n    inp = tf.keras.Input(INPUT_SHAPE)\n    x = tf.keras.layers.Masking(mask_value=0.0)(inp)\n    x = tf.keras.layers.Dense(dim, use_bias=False)(x)\n    \n    #Positional encoding\n    pe = tf.cast(positional_encoding(INPUT_SHAPE[0], dim), dtype=x.dtype)\n    x = x + pe\n    x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n    \n    #CNN-Transformer\n    num_blocks = 6\n    drop_rate  = 0.4\n    for i in range(num_blocks):\n        x = Conv1DBlock(dim, 11, drop_rate=drop_rate)(x)\n        x = Conv1DBlock(dim,  5, drop_rate=drop_rate)(x)\n        x = Conv1DBlock(dim,  3, drop_rate=drop_rate)(x)\n        x = TransformerBlock(dim, expand=2)(x)\n\n    #classifier layers\n    x = tf.keras.layers.Dense(dim*2,activation='relu')(x)\n    x = tf.keras.layers.Dropout(0.4)(x)\n    x = tf.keras.layers.Dense(len(char_to_num))(x)\n\n    model = tf.keras.Model(inp, x)\n\n    loss = CTCLoss\n    \n    #Optimizer\n    optimizer = RectifiedAdam(sma_threshold=4)\n    optimizer = Lookahead(optimizer, sync_period=5)\n\n    model.compile(loss=loss, optimizer=optimizer)\n\n    return model\n\ntf.keras.backend.clear_session()\nmodel = get_model()\nmodel(batch[0])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:25:19.365323Z","iopub.execute_input":"2024-06-04T17:25:19.365688Z","iopub.status.idle":"2024-06-04T17:25:25.340463Z","shell.execute_reply.started":"2024-06-04T17:25:19.365662Z","shell.execute_reply":"2024-06-04T17:25:25.339045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def num_to_char_fn(y):\n    return [num_to_char.get(x, \"\") for x in y]\n\n@tf.function()\ndef decode_phrase(pred):\n    x = tf.argmax(pred, axis=1)\n    diff = tf.not_equal(x[:-1], x[1:])\n    adjacent_indices = tf.where(diff)[:, 0]\n    x = tf.gather(x, adjacent_indices)\n    mask = x != pad_token_idx\n    x = tf.boolean_mask(x, mask, axis=0)\n    return x\n\n# Декодирование входных данных сети\ndef decode_batch_predictions(pred):\n    output_text = []\n    for result in pred:\n        result = \"\".join(num_to_char_fn(decode_phrase(result).numpy()))\n        output_text.append(result)\n    return output_text","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:26:09.640547Z","iopub.execute_input":"2024-06-04T17:26:09.64123Z","iopub.status.idle":"2024-06-04T17:26:09.64934Z","shell.execute_reply.started":"2024-06-04T17:26:09.641198Z","shell.execute_reply":"2024-06-04T17:26:09.648184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CallbackEval(tf.keras.callbacks.Callback):\n    \"\"\"Displays a batch of outputs after every epoch.\"\"\"\n\n    def __init__(self, dataset):\n        super().__init__()\n        self.dataset = dataset\n\n    def on_epoch_end(self, epoch: int, logs=None):\n        model.save_weights(\"model.h5\")\n        predictions = []\n        targets = []\n        for batch in self.dataset:\n            X, y = batch\n            batch_predictions = model(X)\n            batch_predictions = decode_batch_predictions(batch_predictions)\n            predictions.extend(batch_predictions)\n            for label in y:\n                label = \"\".join(num_to_char_fn(label.numpy()))\n                targets.append(label)\n        print(\"-\" * 100)\n        resize_pad\n        for i in range(10):\n            print(f\"Target    : {targets[i]}\")\n            print(f\"Prediction: {predictions[i]}, len: {len(predictions[i])}\")\n            print(\"-\" * 100)\n\n# Функция проверки полученного результата обучения\nvalidation_callback = CallbackEval(val_dataset.take(1))","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:26:11.899934Z","iopub.execute_input":"2024-06-04T17:26:11.900582Z","iopub.status.idle":"2024-06-04T17:26:11.912413Z","shell.execute_reply.started":"2024-06-04T17:26:11.900549Z","shell.execute_reply":"2024-06-04T17:26:11.911497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_INTERACTIVE = False\nN_EPOCHS = 2 if IS_INTERACTIVE else 50\nN_WARMUP_EPOCHS = 0 if IS_INTERACTIVE else 5\nLR_MAX = 4e-3\nWD_RATIO = 0.05\nWARMUP_METHOD = \"exp\"","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:26:12.67057Z","iopub.execute_input":"2024-06-04T17:26:12.67146Z","iopub.status.idle":"2024-06-04T17:26:12.676094Z","shell.execute_reply.started":"2024-06-04T17:26:12.671428Z","shell.execute_reply":"2024-06-04T17:26:12.675082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n    \n    if current_step < num_warmup_steps:\n        if WARMUP_METHOD == 'log':\n            return lr_max * 0.10 ** (num_warmup_steps - current_step)\n        else:\n            return lr_max * 2 ** -(num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max\n    \ndef plot_lr_schedule(lr_schedule, epochs):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(1, epochs + 1)\n    x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n    plt.xlim([1, epochs])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n    # Increase y-limit for better readability\n    plt.ylim([0, max(lr_schedule) * 1.1])\n    \n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n            if x < len(lr_schedule) - 1:\n                if lr_schedule[x - 1] < val:\n                    ha = 'right'\n                else:\n                    ha = 'left'\n            elif x == 0:\n                ha = 'right'\n            else:\n                ha = 'left'\n            plt.plot(x + 1, val, 'o', color='black');\n            offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n            plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=0)\n\n# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:26:14.313296Z","iopub.execute_input":"2024-06-04T17:26:14.314049Z","iopub.status.idle":"2024-06-04T17:26:14.716136Z","shell.execute_reply.started":"2024-06-04T17:26:14.314007Z","shell.execute_reply":"2024-06-04T17:26:14.71517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    train_dataset,\n    validation_data=val_dataset,\n    epochs=N_EPOCHS,\n    callbacks=[\n        validation_callback,\n        lr_callback,\n        WeightDecayCallback(),\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:26:16.804648Z","iopub.execute_input":"2024-06-04T17:26:16.805004Z","iopub.status.idle":"2024-06-04T17:46:30.25226Z","shell.execute_reply.started":"2024-06-04T17:26:16.804977Z","shell.execute_reply":"2024-06-04T17:46:30.251395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train_loss', 'val_loss'], loc='upper right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:46:41.969333Z","iopub.execute_input":"2024-06-04T17:46:41.970051Z","iopub.status.idle":"2024-06-04T17:46:42.240891Z","shell.execute_reply.started":"2024-06-04T17:46:41.970005Z","shell.execute_reply":"2024-06-04T17:46:42.239892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n        self.model = model\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(SEL_COLS)], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs, training=False):\n        # Preprocess Data\n        x = tf.cast(inputs, tf.float32)\n        x = x[None]\n        x = tf.cond(tf.shape(x)[1] == 0, lambda: tf.zeros((1, 1, len(SEL_COLS))), lambda: tf.identity(x))\n        x = x[0]\n        x = pre_process_data(x)\n        x = pre_process_shape(*x)\n        x = tf.reshape(x, INPUT_SHAPE)\n        x = x[None]\n        x = self.model(x, training=False)\n        x = x[0]\n        x = decode_phrase(x)\n\n        x = tf.cond(tf.shape(x)[0] < 3, lambda: tf.constant(\n            [17, 0, 32, 12, 36, 0, 12, 32, 49, 46, 36], tf.int64), lambda: tf.identity(x))\n        x = tf.one_hot(x, 59)\n        return {'outputs': x}\n\ntflitemodel_base = TFLiteModel(model)\ntflitemodel_base(frames)[\"outputs\"].shape","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:46:46.210421Z","iopub.execute_input":"2024-06-04T17:46:46.211167Z","iopub.status.idle":"2024-06-04T17:46:50.264048Z","shell.execute_reply.started":"2024-06-04T17:46:46.211131Z","shell.execute_reply":"2024-06-04T17:46:50.263169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]#, tf.lite.OpsSet.SELECT_TF_OPS]\nkeras_model_converter.optimizations = [tf.lite.Optimize.DEFAULT]\nkeras_model_converter.target_spec.supported_types = [tf.float16]\ntflite_model = keras_model_converter.convert()\nwith open('model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \nwith open('inference_args.json', \"w\") as f:\n    json.dump({\"selected_columns\" : SEL_COLS}, f)\n    ","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:47:00.454539Z","iopub.execute_input":"2024-06-04T17:47:00.455425Z","iopub.status.idle":"2024-06-04T17:47:58.550381Z","shell.execute_reply.started":"2024-06-04T17:47:00.455389Z","shell.execute_reply":"2024-06-04T17:47:58.54932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"inference_args.json\", \"r\") as f:\n    SEL_COLS = json.load(f)[\"selected_columns\"]\n    \ndef load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=SEL_COLS)\n\ndef create_data_gen(file_ids, y_mul=1):\n    def gen():\n        for file_id in file_ids:\n            pqfile = f\"{inpdir}/{file_id}.parquet\"\n            seq_refs = df.loc[df.file_id == file_id]\n            seqs = load_relevant_data_subset(pqfile)\n\n            for seq_id in seq_refs.sequence_id:\n                x = seqs.iloc[seqs.index == seq_id].to_numpy()\n                y = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])\n                \n                r_nonan = np.sum(np.sum(np.isnan(x[:, RHAND_IDX_X]), axis = 1) == 0)\n                l_nonan = np.sum(np.sum(np.isnan(x[:, LHAND_IDX_X]), axis = 1) == 0)\n                no_nan = max(r_nonan, l_nonan)\n                \n                if y_mul*len(y)<no_nan:\n                    yield x, y\n    return gen\n\npqfiles = df.file_id.unique()\nval_len = int(0.05 * len(pqfiles))\n\ntest_dataset = tf.data.Dataset.from_generator(create_data_gen(pqfiles[:val_len], 0),\n    output_signature=(tf.TensorSpec(shape=(None, len(SEL_COLS)), dtype=tf.float32), tf.TensorSpec(shape=(), dtype=tf.string))\n).prefetch(buffer_size=2000)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:48:15.728914Z","iopub.execute_input":"2024-06-04T17:48:15.729843Z","iopub.status.idle":"2024-06-04T17:48:15.773031Z","shell.execute_reply.started":"2024-06-04T17:48:15.729807Z","shell.execute_reply":"2024-06-04T17:48:15.772088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tf.lite.Interpreter(\"model.tflite\")\n\nREQUIRED_SIGNATURE = \"serving_default\"\nREQUIRED_OUTPUT = \"outputs\"\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nprediction_fn = interpreter.get_signature_runner(REQUIRED_SIGNATURE)\n\nfor frame, target in test_dataset.skip(100).take(10):\n    output = prediction_fn(inputs=frame)\n    prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\n    target = target.numpy().decode(\"utf-8\")\n    print(\"pred =\", prediction_str, \"; target =\", target)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:48:18.735464Z","iopub.execute_input":"2024-06-04T17:48:18.736381Z","iopub.status.idle":"2024-06-04T17:48:24.216424Z","shell.execute_reply.started":"2024-06-04T17:48:18.736345Z","shell.execute_reply":"2024-06-04T17:48:24.215563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install Levenshtein\nfrom Levenshtein import distance\n\nscores = []\n\nfor i, (frame, target) in tqdm(enumerate(test_dataset.take(1000))):\n    output = prediction_fn(inputs=frame)\n    prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\n    target = target.numpy().decode(\"utf-8\")\n    score = (len(target) - distance(prediction_str, target)) / len(target)\n    scores.append(score)\n    if i % 50 == 0:\n        print(np.sum(scores) / len(scores))\n    \nscores = np.array(scores)\nprint(np.sum(scores) / len(scores))","metadata":{"execution":{"iopub.status.busy":"2024-06-04T17:49:43.060653Z","iopub.execute_input":"2024-06-04T17:49:43.061492Z","iopub.status.idle":"2024-06-04T17:53:22.264012Z","shell.execute_reply.started":"2024-06-04T17:49:43.061452Z","shell.execute_reply":"2024-06-04T17:53:22.262945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}