{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is base on [SKNADIG's notebook](https://www.kaggle.com/code/nadigshreekanth/data-visualization-using-mediapipe-apis)","metadata":{}},{"cell_type":"markdown","source":"## Install MediaPipe","metadata":{}},{"cell_type":"code","source":"!pip install mediapipe","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:40:12.369369Z","iopub.execute_input":"2023-07-07T10:40:12.369677Z","iopub.status.idle":"2023-07-07T10:40:26.748915Z","shell.execute_reply.started":"2023-07-07T10:40:12.369651Z","shell.execute_reply":"2023-07-07T10:40:26.747554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Libraries","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nimport tensorflow as tf\nimport json\nimport mediapipe\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport random\nfrom pathlib import Path\n\n\nfrom skimage.transform import resize\nfrom mediapipe.framework.formats import landmark_pb2\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tqdm.notebook import tqdm\nfrom matplotlib import animation, rc\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-07T10:40:35.96388Z","iopub.execute_input":"2023-07-07T10:40:35.964477Z","iopub.status.idle":"2023-07-07T10:40:35.975171Z","shell.execute_reply.started":"2023-07-07T10:40:35.964448Z","shell.execute_reply":"2023-07-07T10:40:35.974238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"TensorFlow v\" + tf.__version__)\nprint(\"Mediapipe v\" + mediapipe.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:40:35.977672Z","iopub.execute_input":"2023-07-07T10:40:35.978224Z","iopub.status.idle":"2023-07-07T10:40:36.000722Z","shell.execute_reply.started":"2023-07-07T10:40:35.978187Z","shell.execute_reply":"2023-07-07T10:40:35.999386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load the dataset","metadata":{}},{"cell_type":"code","source":"base_dir = Path('/kaggle/input/asl-fingerspelling/')\n\nfrom matplotlib import animation, rc\nrc('animation', html='jshtml')\n\ndef create_animation(images):\n    fig = plt.figure(figsize=(6, 9))\n    ax = plt.Axes(fig, [0., 0., 1., 1.])\n    ax.set_axis_off()\n    fig.add_axes(ax)\n    im=ax.imshow(images[0], cmap=\"gray\")\n    plt.close(fig)\n    \n    def animate_func(i):\n        im.set_array(images[i])\n        return [im]\n\n    return animation.FuncAnimation(fig, animate_func, frames=len(images), interval=1000/10)\n\ntrain_df = pd.read_csv(base_dir / 'train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:40:46.078874Z","iopub.execute_input":"2023-07-07T10:40:46.079233Z","iopub.status.idle":"2023-07-07T10:40:46.173258Z","shell.execute_reply.started":"2023-07-07T10:40:46.079204Z","shell.execute_reply":"2023-07-07T10:40:46.172361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Full train dataset shape is {}\".format(train_df.shape))","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:41:42.276483Z","iopub.execute_input":"2023-07-07T10:41:42.277294Z","iopub.status.idle":"2023-07-07T10:41:42.283574Z","shell.execute_reply.started":"2023-07-07T10:41:42.277257Z","shell.execute_reply":"2023-07-07T10:41:42.282553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:41:45.803688Z","iopub.execute_input":"2023-07-07T10:41:45.804891Z","iopub.status.idle":"2023-07-07T10:41:45.837158Z","shell.execute_reply.started":"2023-07-07T10:41:45.804847Z","shell.execute_reply":"2023-07-07T10:41:45.836159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fetch sequence_id, file_id, phrase from first row\nsequence_id, file_id, phrase = train_df.iloc[0][['sequence_id', 'file_id', 'phrase']]\nprint(f\"sequence_id: {sequence_id}, file_id: {file_id}, phrase: {phrase}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:41:58.937195Z","iopub.execute_input":"2023-07-07T10:41:58.937607Z","iopub.status.idle":"2023-07-07T10:41:58.950397Z","shell.execute_reply.started":"2023-07-07T10:41:58.937575Z","shell.execute_reply":"2023-07-07T10:41:58.949553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fetch data from parquet file\nsample_sequence_df = pq.read_table(f\"/kaggle/input/asl-fingerspelling/train_landmarks/{str(file_id)}.parquet\",\n    filters=[[('sequence_id', '=', sequence_id)],]).to_pandas()\nprint(\"Full sequence dataset shape is {}\".format(sample_sequence_df.shape))","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:42:10.016822Z","iopub.execute_input":"2023-07-07T10:42:10.017241Z","iopub.status.idle":"2023-07-07T10:42:13.443024Z","shell.execute_reply.started":"2023-07-07T10:42:10.017208Z","shell.execute_reply":"2023-07-07T10:42:13.441663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sequence_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:42:16.361606Z","iopub.execute_input":"2023-07-07T10:42:16.361998Z","iopub.status.idle":"2023-07-07T10:42:16.388063Z","shell.execute_reply.started":"2023-07-07T10:42:16.361967Z","shell.execute_reply":"2023-07-07T10:42:16.386964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_random_sequence(phrase_idx=None):\n    if phrase_idx == None:\n        phrase_idx = np.random.randint(len(train_df))\n\n    selected_row = train_df.iloc[phrase_idx]\n    file_id = selected_row['file_id']\n    sequence_id = selected_row['sequence_id']\n    phrase = selected_row['phrase']\n    \n    parquet_file = pq.ParquetFile(base_dir / 'train_landmarks' / f\"{str(file_id)}.parquet\")\n    \n    dataset = pq.read_table(\n        base_dir / 'train_landmarks' / f\"{str(file_id)}.parquet\",\n        filters=[\n            [('sequence_id', '=', sequence_id)],\n        ]\n    )\n    \n    sequence_df = dataset.to_pandas()\n    return phrase_idx, phrase, sequence_df","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:42:59.358561Z","iopub.execute_input":"2023-07-07T10:42:59.359318Z","iopub.status.idle":"2023-07-07T10:42:59.365877Z","shell.execute_reply.started":"2023-07-07T10:42:59.359285Z","shell.execute_reply":"2023-07-07T10:42:59.364774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"phrase_idx = 2000\nselected_row = train_df.iloc[phrase_idx]\nfile_id = selected_row['file_id']\nsequence_id = selected_row['sequence_id']\nphrase = selected_row['phrase']\nprint(f\"Phrase: {phrase}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:47:06.093712Z","iopub.execute_input":"2023-07-07T10:47:06.094076Z","iopub.status.idle":"2023-07-07T10:47:06.100059Z","shell.execute_reply.started":"2023-07-07T10:47:06.094047Z","shell.execute_reply":"2023-07-07T10:47:06.099013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquet_file = pq.ParquetFile(base_dir / 'train_landmarks' / f\"{str(file_id)}.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:47:11.128566Z","iopub.execute_input":"2023-07-07T10:47:11.129596Z","iopub.status.idle":"2023-07-07T10:47:11.164092Z","shell.execute_reply.started":"2023-07-07T10:47:11.129557Z","shell.execute_reply":"2023-07-07T10:47:11.163253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read only the relevant rows for this sequence ID. Might come handy when creating a data loader for model training","metadata":{}},{"cell_type":"code","source":"dataset = pq.read_table(\n    base_dir / 'train_landmarks' / f\"{str(file_id)}.parquet\",\n    filters=[\n        [('sequence_id', '=', sequence_id)],\n    ]\n)\n\nsequence_df = dataset.to_pandas()\nsequence_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:48:25.616586Z","iopub.execute_input":"2023-07-07T10:48:25.617038Z","iopub.status.idle":"2023-07-07T10:48:27.02061Z","shell.execute_reply.started":"2023-07-07T10:48:25.617007Z","shell.execute_reply":"2023-07-07T10:48:27.019571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Mediapipe helpers","metadata":{}},{"cell_type":"code","source":"mp_pose = mediapipe.solutions.pose\nmp_hands = mediapipe.solutions.hands\nmp_face_mesh = mediapipe.solutions.face_mesh\nmp_drawing = mediapipe.solutions.drawing_utils\nmp_drawing_styles = mediapipe.solutions.drawing_styles","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:59:23.138019Z","iopub.execute_input":"2023-07-07T10:59:23.138452Z","iopub.status.idle":"2023-07-07T10:59:23.144895Z","shell.execute_reply.started":"2023-07-07T10:59:23.138417Z","shell.execute_reply":"2023-07-07T10:59:23.143518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hands(seq_df):\n    images = []\n    all_hand_landmarks = []\n    for seq_idx in range(len(seq_df)):\n        x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_right_hand.*\").values\n        y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_right_hand.*\").values\n        z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_right_hand.*\").values\n\n        right_hand_image = np.zeros((900, 600, 3))\n\n        right_hand_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for x, y, z in zip(x_pose, y_pose, z_pose):\n            right_hand_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n                right_hand_image,\n                right_hand_landmarks,\n                mp_hands.HAND_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_left_hand.*\").values\n        y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_left_hand.*\").values\n        z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_left_hand.*\").values\n        \n        left_hand_image = np.zeros((900, 600, 3))\n        \n        left_hand_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for x, y, z in zip(x_pose, y_pose, z_pose):\n            left_hand_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n                left_hand_image,\n                left_hand_landmarks,\n                mp_hands.HAND_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        images.append([right_hand_image.astype(np.uint8), left_hand_image.astype(np.uint8)])\n        all_hand_landmarks.append([right_hand_landmarks, left_hand_landmarks])\n    return images, all_hand_landmarks","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:59:25.695592Z","iopub.execute_input":"2023-07-07T10:59:25.696292Z","iopub.status.idle":"2023-07-07T10:59:25.709926Z","shell.execute_reply.started":"2023-07-07T10:59:25.696246Z","shell.execute_reply":"2023-07-07T10:59:25.709043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hand_images, hand_landmarks = get_hands(sequence_df)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T10:59:30.154119Z","iopub.execute_input":"2023-07-07T10:59:30.154914Z","iopub.status.idle":"2023-07-07T10:59:31.461146Z","shell.execute_reply.started":"2023-07-07T10:59:30.154873Z","shell.execute_reply":"2023-07-07T10:59:31.459901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Right hand","metadata":{}},{"cell_type":"code","source":"create_animation(np.array(hand_images)[:, 0])","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:03:32.000884Z","iopub.execute_input":"2023-07-07T11:03:32.00133Z","iopub.status.idle":"2023-07-07T11:03:36.648612Z","shell.execute_reply.started":"2023-07-07T11:03:32.001299Z","shell.execute_reply":"2023-07-07T11:03:36.647583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Left hand","metadata":{}},{"cell_type":"code","source":"create_animation(np.array(hand_images)[:, 1])","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:04:22.208941Z","iopub.execute_input":"2023-07-07T11:04:22.209363Z","iopub.status.idle":"2023-07-07T11:04:26.731688Z","shell.execute_reply.started":"2023-07-07T11:04:22.209329Z","shell.execute_reply":"2023-07-07T11:04:26.730928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Face","metadata":{}},{"cell_type":"code","source":"def get_face(seq_df):\n    images = []\n    all_face_landmarks = []\n    for seq_idx in range(len(seq_df)):\n        x_face = seq_df.iloc[seq_idx].filter(regex=\"x_face.*\").values\n        y_face = seq_df.iloc[seq_idx].filter(regex=\"y_face.*\").values\n        z_face = seq_df.iloc[seq_idx].filter(regex=\"z_face.*\").values\n\n        annotated_image = np.zeros((900, 600, 3))\n\n        face_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for x, y, z in zip(x_face, y_face, z_face):\n            face_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n          image=annotated_image,\n          landmark_list=face_landmarks,\n          connections=mp_face_mesh.FACEMESH_TESSELATION,\n          landmark_drawing_spec=None,\n          connection_drawing_spec=mp_drawing_styles\n          .get_default_face_mesh_tesselation_style())\n        mp_drawing.draw_landmarks(\n          image=annotated_image,\n          landmark_list=face_landmarks,\n          connections=mp_face_mesh.FACEMESH_CONTOURS,\n          landmark_drawing_spec=None,\n          connection_drawing_spec=mp_drawing_styles\n          .get_default_face_mesh_contours_style())\n\n        images.append(annotated_image.astype(np.uint8))\n        all_face_landmarks.append(face_landmarks)\n    return images, all_face_landmarks","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:06:22.281053Z","iopub.execute_input":"2023-07-07T11:06:22.281427Z","iopub.status.idle":"2023-07-07T11:06:22.290506Z","shell.execute_reply.started":"2023-07-07T11:06:22.281399Z","shell.execute_reply":"2023-07-07T11:06:22.289197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"face_images, face_landmarks = get_face(sequence_df)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:06:25.265377Z","iopub.execute_input":"2023-07-07T11:06:25.265788Z","iopub.status.idle":"2023-07-07T11:06:26.337168Z","shell.execute_reply.started":"2023-07-07T11:06:25.265755Z","shell.execute_reply":"2023-07-07T11:06:26.336132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_animation(face_images)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:06:43.249336Z","iopub.execute_input":"2023-07-07T11:06:43.249718Z","iopub.status.idle":"2023-07-07T11:06:48.115652Z","shell.execute_reply.started":"2023-07-07T11:06:43.249673Z","shell.execute_reply":"2023-07-07T11:06:48.114478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pose","metadata":{}},{"cell_type":"code","source":"def get_pose(seq_df):\n    images = []\n    all_pose_landmarks = []\n    for seq_idx in range(len(seq_df)):\n        x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_pose.*\").values\n        y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_pose.*\").values\n        z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_pose.*\").values\n\n        annotated_image = np.zeros((900, 600, 3))\n        \n        data_points = []\n        for x, y, z in zip(x_pose, y_pose, z_pose):\n            data_points.append(np.array([x, y, z]))\n\n        pose_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for row in data_points:\n            pose_landmarks.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                annotated_image,\n                pose_landmarks,\n                mp_pose.POSE_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())\n        images.append(annotated_image.astype(np.uint8))\n        all_pose_landmarks.append(pose_landmarks)\n    return images, all_pose_landmarks\n\npose_images, pose_landmarks = get_pose(sequence_df)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:08:16.854461Z","iopub.execute_input":"2023-07-07T11:08:16.855429Z","iopub.status.idle":"2023-07-07T11:08:17.517708Z","shell.execute_reply.started":"2023-07-07T11:08:16.855375Z","shell.execute_reply":"2023-07-07T11:08:17.516537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_animation(pose_images)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:08:45.652387Z","iopub.execute_input":"2023-07-07T11:08:45.652905Z","iopub.status.idle":"2023-07-07T11:08:50.306794Z","shell.execute_reply.started":"2023-07-07T11:08:45.652856Z","shell.execute_reply":"2023-07-07T11:08:50.305739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_landmark_to_npy(landmarklist):\n    return np.array([np.array([landmark.x, landmark.y, landmark.z]) for landmark in landmarklist.landmark])","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:09:46.223162Z","iopub.execute_input":"2023-07-07T11:09:46.223677Z","iopub.status.idle":"2023-07-07T11:09:46.229337Z","shell.execute_reply.started":"2023-07-07T11:09:46.223645Z","shell.execute_reply":"2023-07-07T11:09:46.228101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_all_images(seq_df):\n    pose_images, pose_landmarks = get_pose(sequence_df)\n    hand_images, hand_landmarks = get_hands(sequence_df)\n    face_images, face_landmarks = get_face(sequence_df)\n    \n    all_images = []\n    all_landmarks_data = []\n    all_landmarks = []\n    for seq_idx in tqdm(range(len(pose_landmarks))):\n        pose_landmark_np = convert_landmark_to_npy(pose_landmarks[seq_idx])\n        right_hand_landmark_np = convert_landmark_to_npy(hand_landmarks[seq_idx][0])\n        left_hand_landmark_np = convert_landmark_to_npy(hand_landmarks[seq_idx][1])\n        face_landmark_np = convert_landmark_to_npy(face_landmarks[seq_idx])\n        \n        # Pool all landmarks together to find min and max coordinates\n        pooled_landmarks = np.vstack((pose_landmark_np, right_hand_landmark_np, left_hand_landmark_np, face_landmark_np))\n        pooled_min = np.nanmin(pooled_landmarks, axis=0)\n        pooled_max = np.nanmax(pooled_landmarks, axis=0)\n        \n        # Use the max of x and y scaling to proportionally scale the image. We don't need to scale z for 2D image\n        # There is an elegant way to achieve the same result as below with Matrix transformations. Might re-do this if there is interest.\n        scaling_factor = np.nanmax(pooled_max[:2])\n        pooled_scaled_min = np.nanmin(pooled_landmarks / scaling_factor, axis=0)\n\n        pose_landmark_np_normed = (pose_landmark_np / scaling_factor) - pooled_scaled_min\n        \n        # Center the image around shoulder and hips. Makes for a better visualization\n        x_shift = ((1-(pose_landmark_np_normed[23]+pose_landmark_np_normed[24]))/2)[0]\n        axis_shift = np.array([x_shift, 0, 0])\n        \n        pose_landmark_np_normed = pose_landmark_np_normed + axis_shift\n        right_hand_landmark_np_normed = (right_hand_landmark_np / scaling_factor) - pooled_scaled_min + axis_shift\n        left_hand_landmark_np_normed = (left_hand_landmark_np / scaling_factor) - pooled_scaled_min  + axis_shift\n        face_landmark_np_normed = (face_landmark_np / scaling_factor) - pooled_scaled_min + axis_shift\n        \n        # Now that we have scaled and shifted the landmarks to fit into a [0, 1] range, we can start plotting them using mediapipe APIs\n        # BG image with zeros\n        image = np.zeros((900, 600, 3))\n        \n        # Pose\n        pose_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in pose_landmark_np_normed:\n            pose_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                    image,\n                    pose_landmark_np_normed_z,\n                    mp_pose.POSE_CONNECTIONS,\n                    landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())\n\n        # Right hand\n        right_hand_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in right_hand_landmark_np_normed:\n            right_hand_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                    image,\n                    right_hand_landmark_np_normed_z,\n                    mp_hands.HAND_CONNECTIONS,\n                    landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        # Left hand\n        left_hand_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in left_hand_landmark_np_normed:\n            left_hand_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                    image,\n                    left_hand_landmark_np_normed_z,\n                    mp_hands.HAND_CONNECTIONS,\n                    landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        # Face\n        face_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in face_landmark_np_normed:\n            face_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n            image=image,\n            landmark_list=face_landmark_np_normed_z,\n            connections=mp_face_mesh.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles\n            .get_default_face_mesh_tesselation_style())\n        \n        mp_drawing.draw_landmarks(\n            image=image,\n            landmark_list=face_landmark_np_normed_z,\n            connections=mp_face_mesh.FACEMESH_CONTOURS,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles\n            .get_default_face_mesh_contours_style())\n        \n        # Iris data not available. So ignoring the iris visualization.\n        \n        all_images.append(image.astype(np.uint8))\n        all_landmarks_data.append([pose_landmark_np_normed, right_hand_landmark_np_normed_z, left_hand_landmark_np_normed_z, face_landmark_np_normed_z])\n        all_landmarks.append([pose_landmark_np_normed_z, right_hand_landmark_np_normed_z, left_hand_landmark_np_normed_z, face_landmark_np_normed_z])\n    return all_images, all_landmarks_data, all_landmarks","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:09:49.024097Z","iopub.execute_input":"2023-07-07T11:09:49.024476Z","iopub.status.idle":"2023-07-07T11:09:49.041775Z","shell.execute_reply.started":"2023-07-07T11:09:49.024447Z","shell.execute_reply":"2023-07-07T11:09:49.040475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"phrase_idx, phrase, sequence_df = get_random_sequence(0)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:10:11.682468Z","iopub.execute_input":"2023-07-07T11:10:11.683378Z","iopub.status.idle":"2023-07-07T11:10:14.47021Z","shell.execute_reply.started":"2023-07-07T11:10:11.68334Z","shell.execute_reply":"2023-07-07T11:10:14.468445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"{phrase_idx}: {phrase}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:10:15.0806Z","iopub.execute_input":"2023-07-07T11:10:15.081029Z","iopub.status.idle":"2023-07-07T11:10:15.087195Z","shell.execute_reply.started":"2023-07-07T11:10:15.080995Z","shell.execute_reply":"2023-07-07T11:10:15.086217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_images, all_landmarks_data, all_landmarks = get_all_images(sequence_df)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:10:28.739091Z","iopub.execute_input":"2023-07-07T11:10:28.739525Z","iopub.status.idle":"2023-07-07T11:10:38.461661Z","shell.execute_reply.started":"2023-07-07T11:10:28.739494Z","shell.execute_reply":"2023-07-07T11:10:38.460891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_animation(all_images)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:10:58.956883Z","iopub.execute_input":"2023-07-07T11:10:58.957292Z","iopub.status.idle":"2023-07-07T11:11:12.914251Z","shell.execute_reply.started":"2023-07-07T11:10:58.957261Z","shell.execute_reply":"2023-07-07T11:11:12.91317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fill missing values with previous values","metadata":{}},{"cell_type":"code","source":"all_images2, _, _ = get_all_images(sequence_df.fillna(method='ffill'))","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:12:14.596562Z","iopub.execute_input":"2023-07-07T11:12:14.596973Z","iopub.status.idle":"2023-07-07T11:12:23.51478Z","shell.execute_reply.started":"2023-07-07T11:12:14.59694Z","shell.execute_reply":"2023-07-07T11:12:23.513938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_animation(all_images2)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:12:33.321262Z","iopub.execute_input":"2023-07-07T11:12:33.321727Z","iopub.status.idle":"2023-07-07T11:12:47.427266Z","shell.execute_reply.started":"2023-07-07T11:12:33.321682Z","shell.execute_reply":"2023-07-07T11:12:47.42601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3D Plot","metadata":{}},{"cell_type":"code","source":"mp_drawing.plot_landmarks(all_landmarks[0][0], mp_pose.POSE_CONNECTIONS, azimuth=0, elevation=0)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:13:17.290854Z","iopub.execute_input":"2023-07-07T11:13:17.291268Z","iopub.status.idle":"2023-07-07T11:13:17.988367Z","shell.execute_reply.started":"2023-07-07T11:13:17.291237Z","shell.execute_reply":"2023-07-07T11:13:17.987281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mp_drawing.plot_landmarks(all_landmarks[0][0], mp_pose.POSE_CONNECTIONS, azimuth=90, elevation=0)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T11:14:53.164769Z","iopub.execute_input":"2023-07-07T11:14:53.16517Z","iopub.status.idle":"2023-07-07T11:14:53.863516Z","shell.execute_reply.started":"2023-07-07T11:14:53.165138Z","shell.execute_reply":"2023-07-07T11:14:53.862456Z"},"trusted":true},"execution_count":null,"outputs":[]}]}