{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### This is a notebook I created to explore the dataset and visualize the landmarks to get a feel of the problem.\n### The code uses mediapipe APIs where available.","metadata":{}},{"cell_type":"markdown","source":"# Reference materials for this notebook\n* https://developers.google.com/mediapipe/solutions/guide\n* Pose: https://colab.research.google.com/github/googlesamples/mediapipe/blob/main/examples/pose_landmarker/python/%5BMediaPipe_Python_Tasks%5D_Pose_Landmarker.ipynb\n* Hand: https://colab.research.google.com/github/googlesamples/mediapipe/blob/main/examples/hand_landmarker/python/hand_landmarker.ipynb\n* Face: https://colab.research.google.com/github/googlesamples/mediapipe/blob/main/examples/face_landmarker/python/%5BMediaPipe_Python_Tasks%5D_Face_Landmarker.ipynb\n* mediapipe Python bindings: https://github.com/google/mediapipe/tree/master/mediapipe/python\n* mediapipe samples: https://github.com/googlesamples/mediapipe","metadata":{}},{"cell_type":"code","source":"!pip install mediapipe","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:18.920342Z","iopub.execute_input":"2023-06-03T05:48:18.92069Z","iopub.status.idle":"2023-06-03T05:48:30.382477Z","shell.execute_reply.started":"2023-06-03T05:48:18.920661Z","shell.execute_reply":"2023-06-03T05:48:30.38157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nfrom mediapipe.framework.formats import landmark_pb2\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nimport mediapipe\nfrom mediapipe.framework.formats import landmark_pb2\nfrom tqdm.notebook import tqdm\nimport matplotlib\nmatplotlib.rcParams['animation.embed_limit'] = 2**128\nmatplotlib.rcParams['savefig.pad_inches'] = 0","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:30.384252Z","iopub.execute_input":"2023-06-03T05:48:30.384552Z","iopub.status.idle":"2023-06-03T05:48:38.774506Z","shell.execute_reply.started":"2023-06-03T05:48:30.384524Z","shell.execute_reply":"2023-06-03T05:48:38.773527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_dir = Path('/kaggle/input/asl-fingerspelling/')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-03T05:48:38.775569Z","iopub.execute_input":"2023-06-03T05:48:38.776284Z","iopub.status.idle":"2023-06-03T05:48:38.781925Z","shell.execute_reply.started":"2023-06-03T05:48:38.776253Z","shell.execute_reply":"2023-06-03T05:48:38.780918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import animation, rc\nrc('animation', html='jshtml')\n\ndef create_animation(images):\n    fig = plt.figure(figsize=(6, 9))\n    ax = plt.Axes(fig, [0., 0., 1., 1.])\n    ax.set_axis_off()\n    fig.add_axes(ax)\n    im=ax.imshow(images[0], cmap=\"gray\")\n    plt.close(fig)\n    \n    def animate_func(i):\n        im.set_array(images[i])\n        return [im]\n\n    return animation.FuncAnimation(fig, animate_func, frames=len(images), interval=1000/10)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:38.784017Z","iopub.execute_input":"2023-06-03T05:48:38.784568Z","iopub.status.idle":"2023-06-03T05:48:38.804574Z","shell.execute_reply.started":"2023-06-03T05:48:38.784538Z","shell.execute_reply":"2023-06-03T05:48:38.803702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(base_dir / 'train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:38.805627Z","iopub.execute_input":"2023-06-03T05:48:38.806587Z","iopub.status.idle":"2023-06-03T05:48:38.917915Z","shell.execute_reply.started":"2023-06-03T05:48:38.806555Z","shell.execute_reply":"2023-06-03T05:48:38.91699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:38.919192Z","iopub.execute_input":"2023-06-03T05:48:38.919463Z","iopub.status.idle":"2023-06-03T05:48:38.947067Z","shell.execute_reply.started":"2023-06-03T05:48:38.919441Z","shell.execute_reply":"2023-06-03T05:48:38.946211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_random_sequence(phrase_idx=None):\n    if phrase_idx == None:\n        phrase_idx = np.random.randint(len(train_df))\n\n    selected_row = train_df.iloc[phrase_idx]\n    file_id = selected_row['file_id']\n    sequence_id = selected_row['sequence_id']\n    phrase = selected_row['phrase']\n    \n    parquet_file = pq.ParquetFile(base_dir / 'train_landmarks' / f\"{str(file_id)}.parquet\")\n    \n    dataset = pq.read_table(\n        base_dir / 'train_landmarks' / f\"{str(file_id)}.parquet\",\n        filters=[\n            [('sequence_id', '=', sequence_id)],\n        ]\n    )\n    \n    sequence_df = dataset.to_pandas()\n    return phrase_idx, phrase, sequence_df","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:38.948213Z","iopub.execute_input":"2023-06-03T05:48:38.948485Z","iopub.status.idle":"2023-06-03T05:48:38.954563Z","shell.execute_reply.started":"2023-06-03T05:48:38.94846Z","shell.execute_reply":"2023-06-03T05:48:38.953678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"phrase_idx = 3\nselected_row = train_df.iloc[phrase_idx]\nfile_id = selected_row['file_id']\nsequence_id = selected_row['sequence_id']\nphrase = selected_row['phrase']\nprint(f\"Phrase: {phrase}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:38.955711Z","iopub.execute_input":"2023-06-03T05:48:38.956337Z","iopub.status.idle":"2023-06-03T05:48:38.967088Z","shell.execute_reply.started":"2023-06-03T05:48:38.956314Z","shell.execute_reply":"2023-06-03T05:48:38.966436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquet_file = pq.ParquetFile(base_dir / 'train_landmarks' / f\"{str(file_id)}.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:38.968086Z","iopub.execute_input":"2023-06-03T05:48:38.968811Z","iopub.status.idle":"2023-06-03T05:48:39.020189Z","shell.execute_reply.started":"2023-06-03T05:48:38.968787Z","shell.execute_reply":"2023-06-03T05:48:39.018867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read only the relevant rows for this sequence ID. Might come handy when creating a data loader for model training","metadata":{}},{"cell_type":"code","source":"dataset = pq.read_table(\n    base_dir / 'train_landmarks' / f\"{str(file_id)}.parquet\",\n    filters=[\n        [('sequence_id', '=', sequence_id)],\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:39.025322Z","iopub.execute_input":"2023-06-03T05:48:39.025592Z","iopub.status.idle":"2023-06-03T05:48:41.979488Z","shell.execute_reply.started":"2023-06-03T05:48:39.025568Z","shell.execute_reply":"2023-06-03T05:48:41.978562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_df = dataset.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:41.980536Z","iopub.execute_input":"2023-06-03T05:48:41.980816Z","iopub.status.idle":"2023-06-03T05:48:42.009962Z","shell.execute_reply.started":"2023-06-03T05:48:41.980792Z","shell.execute_reply":"2023-06-03T05:48:42.009317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:42.011265Z","iopub.execute_input":"2023-06-03T05:48:42.012023Z","iopub.status.idle":"2023-06-03T05:48:42.035554Z","shell.execute_reply.started":"2023-06-03T05:48:42.011994Z","shell.execute_reply":"2023-06-03T05:48:42.034042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mediapipe helpers","metadata":{}},{"cell_type":"code","source":"mp_pose = mediapipe.solutions.pose\nmp_hands = mediapipe.solutions.hands\nmp_face_mesh = mediapipe.solutions.face_mesh\nmp_drawing = mediapipe.solutions.drawing_utils \nmp_drawing_styles = mediapipe.solutions.drawing_styles","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:42.037096Z","iopub.execute_input":"2023-06-03T05:48:42.038085Z","iopub.status.idle":"2023-06-03T05:48:42.049421Z","shell.execute_reply.started":"2023-06-03T05:48:42.038052Z","shell.execute_reply":"2023-06-03T05:48:42.047928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hands(seq_df):\n    images = []\n    all_hand_landmarks = []\n    for seq_idx in range(len(seq_df)):\n        x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_right_hand.*\").values\n        y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_right_hand.*\").values\n        z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_right_hand.*\").values\n\n        right_hand_image = np.zeros((900, 600, 3))\n\n        right_hand_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for x, y, z in zip(x_pose, y_pose, z_pose):\n            right_hand_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n                right_hand_image,\n                right_hand_landmarks,\n                mp_hands.HAND_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_left_hand.*\").values\n        y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_left_hand.*\").values\n        z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_left_hand.*\").values\n        \n        left_hand_image = np.zeros((900, 600, 3))\n        \n        left_hand_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for x, y, z in zip(x_pose, y_pose, z_pose):\n            left_hand_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n                left_hand_image,\n                left_hand_landmarks,\n                mp_hands.HAND_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        images.append([right_hand_image.astype(np.uint8), left_hand_image.astype(np.uint8)])\n        all_hand_landmarks.append([right_hand_landmarks, left_hand_landmarks])\n    return images, all_hand_landmarks","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:42.05112Z","iopub.execute_input":"2023-06-03T05:48:42.051434Z","iopub.status.idle":"2023-06-03T05:48:42.063333Z","shell.execute_reply.started":"2023-06-03T05:48:42.051409Z","shell.execute_reply":"2023-06-03T05:48:42.062225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hand_images, hand_landmarks = get_hands(sequence_df)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:42.064794Z","iopub.execute_input":"2023-06-03T05:48:42.065131Z","iopub.status.idle":"2023-06-03T05:48:45.988037Z","shell.execute_reply.started":"2023-06-03T05:48:42.065109Z","shell.execute_reply":"2023-06-03T05:48:45.987059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Right hand","metadata":{}},{"cell_type":"code","source":"create_animation(np.array(hand_images)[:, 0])","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:48:45.989134Z","iopub.execute_input":"2023-06-03T05:48:45.989419Z","iopub.status.idle":"2023-06-03T05:49:05.423681Z","shell.execute_reply.started":"2023-06-03T05:48:45.989395Z","shell.execute_reply":"2023-06-03T05:49:05.422574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Left hand","metadata":{}},{"cell_type":"code","source":"create_animation(np.array(hand_images)[:, 1])","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:49:05.425091Z","iopub.execute_input":"2023-06-03T05:49:05.425362Z","iopub.status.idle":"2023-06-03T05:49:24.06939Z","shell.execute_reply.started":"2023-06-03T05:49:05.425339Z","shell.execute_reply":"2023-06-03T05:49:24.068406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_face(seq_df):\n    images = []\n    all_face_landmarks = []\n    for seq_idx in range(len(seq_df)):\n        x_face = seq_df.iloc[seq_idx].filter(regex=\"x_face.*\").values\n        y_face = seq_df.iloc[seq_idx].filter(regex=\"y_face.*\").values\n        z_face = seq_df.iloc[seq_idx].filter(regex=\"z_face.*\").values\n\n        annotated_image = np.zeros((900, 600, 3))\n\n        face_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for x, y, z in zip(x_face, y_face, z_face):\n            face_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n          image=annotated_image,\n          landmark_list=face_landmarks,\n          connections=mp_face_mesh.FACEMESH_TESSELATION,\n          landmark_drawing_spec=None,\n          connection_drawing_spec=mp_drawing_styles\n          .get_default_face_mesh_tesselation_style())\n        mp_drawing.draw_landmarks(\n          image=annotated_image,\n          landmark_list=face_landmarks,\n          connections=mp_face_mesh.FACEMESH_CONTOURS,\n          landmark_drawing_spec=None,\n          connection_drawing_spec=mp_drawing_styles\n          .get_default_face_mesh_contours_style())\n\n        images.append(annotated_image.astype(np.uint8))\n        all_face_landmarks.append(face_landmarks)\n    return images, all_face_landmarks","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:49:24.070874Z","iopub.execute_input":"2023-06-03T05:49:24.071178Z","iopub.status.idle":"2023-06-03T05:49:24.079608Z","shell.execute_reply.started":"2023-06-03T05:49:24.071154Z","shell.execute_reply":"2023-06-03T05:49:24.078675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"face_images, face_landmarks = get_face(sequence_df)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:49:24.080686Z","iopub.execute_input":"2023-06-03T05:49:24.080954Z","iopub.status.idle":"2023-06-03T05:49:28.298103Z","shell.execute_reply.started":"2023-06-03T05:49:24.080926Z","shell.execute_reply":"2023-06-03T05:49:28.297439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Face","metadata":{}},{"cell_type":"code","source":"create_animation(face_images)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:49:28.299112Z","iopub.execute_input":"2023-06-03T05:49:28.299789Z","iopub.status.idle":"2023-06-03T05:49:48.42372Z","shell.execute_reply.started":"2023-06-03T05:49:28.299761Z","shell.execute_reply":"2023-06-03T05:49:48.42281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pose(seq_df):\n    images = []\n    all_pose_landmarks = []\n    for seq_idx in range(len(seq_df)):\n        x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_pose.*\").values\n        y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_pose.*\").values\n        z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_pose.*\").values\n\n        annotated_image = np.zeros((900, 600, 3))\n        \n        data_points = []\n        for x, y, z in zip(x_pose, y_pose, z_pose):\n            data_points.append(np.array([x, y, z]))\n\n        pose_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for row in data_points:\n            pose_landmarks.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                annotated_image,\n                pose_landmarks,\n                mp_pose.POSE_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())\n        images.append(annotated_image.astype(np.uint8))\n        all_pose_landmarks.append(pose_landmarks)\n    return images, all_pose_landmarks","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:49:48.425006Z","iopub.execute_input":"2023-06-03T05:49:48.425692Z","iopub.status.idle":"2023-06-03T05:49:48.432893Z","shell.execute_reply.started":"2023-06-03T05:49:48.425664Z","shell.execute_reply":"2023-06-03T05:49:48.431992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pose_images, pose_landmarks = get_pose(sequence_df)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:49:48.433853Z","iopub.execute_input":"2023-06-03T05:49:48.434477Z","iopub.status.idle":"2023-06-03T05:49:50.360934Z","shell.execute_reply.started":"2023-06-03T05:49:48.434455Z","shell.execute_reply":"2023-06-03T05:49:50.360053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pose","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:34:22.499227Z","iopub.execute_input":"2023-06-03T05:34:22.499555Z","iopub.status.idle":"2023-06-03T05:34:22.506034Z","shell.execute_reply.started":"2023-06-03T05:34:22.499528Z","shell.execute_reply":"2023-06-03T05:34:22.504308Z"}}},{"cell_type":"markdown","source":"### Notice how the pose does not fit in the image. This is because the landmarks are not normalized to [0, 1].\n### We will fix this next","metadata":{}},{"cell_type":"code","source":"create_animation(pose_images)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:49:50.361917Z","iopub.execute_input":"2023-06-03T05:49:50.362135Z","iopub.status.idle":"2023-06-03T05:50:09.950277Z","shell.execute_reply.started":"2023-06-03T05:49:50.362116Z","shell.execute_reply":"2023-06-03T05:50:09.947054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_landmark_to_npy(landmarklist):\n    return np.array([np.array([landmark.x, landmark.y, landmark.z]) for landmark in landmarklist.landmark])","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:50:09.951408Z","iopub.execute_input":"2023-06-03T05:50:09.952031Z","iopub.status.idle":"2023-06-03T05:50:09.955966Z","shell.execute_reply.started":"2023-06-03T05:50:09.952008Z","shell.execute_reply":"2023-06-03T05:50:09.955329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_all_images(seq_df):\n    pose_images, pose_landmarks = get_pose(sequence_df)\n    hand_images, hand_landmarks = get_hands(sequence_df)\n    face_images, face_landmarks = get_face(sequence_df)\n    \n    all_images = []\n    all_landmarks_data = []\n    all_landmarks = []\n    for seq_idx in tqdm(range(len(pose_landmarks))):\n        pose_landmark_np = convert_landmark_to_npy(pose_landmarks[seq_idx])\n        right_hand_landmark_np = convert_landmark_to_npy(hand_landmarks[seq_idx][0])\n        left_hand_landmark_np = convert_landmark_to_npy(hand_landmarks[seq_idx][1])\n        face_landmark_np = convert_landmark_to_npy(face_landmarks[seq_idx])\n        \n        # Pool all landmarks together to find min and max coordinates\n        pooled_landmarks = np.vstack((pose_landmark_np, right_hand_landmark_np, left_hand_landmark_np, face_landmark_np))\n        pooled_min = np.nanmin(pooled_landmarks, axis=0)\n        pooled_max = np.nanmax(pooled_landmarks, axis=0)\n        \n        # Use the max of x and y scaling to proportionally scale the image. We don't need to scale z for 2D image\n        # There is an elegant way to achieve the same result as below with Matrix transformations. Might re-do this if there is interest.\n        scaling_factor = np.nanmax(pooled_max[:2])\n        pooled_scaled_min = np.nanmin(pooled_landmarks / scaling_factor, axis=0)\n\n        pose_landmark_np_normed = (pose_landmark_np / scaling_factor) - pooled_scaled_min\n        \n        # Center the image around shoulder and hips. Makes for a better visualization\n        x_shift = ((1-(pose_landmark_np_normed[23]+pose_landmark_np_normed[24]))/2)[0]\n        axis_shift = np.array([x_shift, 0, 0])\n        \n        pose_landmark_np_normed = pose_landmark_np_normed + axis_shift\n        right_hand_landmark_np_normed = (right_hand_landmark_np / scaling_factor) - pooled_scaled_min + axis_shift\n        left_hand_landmark_np_normed = (left_hand_landmark_np / scaling_factor) - pooled_scaled_min  + axis_shift\n        face_landmark_np_normed = (face_landmark_np / scaling_factor) - pooled_scaled_min + axis_shift\n        \n        # Now that we have scaled and shifted the landmarks to fit into a [0, 1] range, we can start plotting them using mediapipe APIs\n        # BG image with zeros\n        image = np.zeros((900, 600, 3))\n        \n        # Pose\n        pose_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in pose_landmark_np_normed:\n            pose_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                    image,\n                    pose_landmark_np_normed_z,\n                    mp_pose.POSE_CONNECTIONS,\n                    landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())\n\n        # Right hand\n        right_hand_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in right_hand_landmark_np_normed:\n            right_hand_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                    image,\n                    right_hand_landmark_np_normed_z,\n                    mp_hands.HAND_CONNECTIONS,\n                    landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        # Left hand\n        left_hand_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in left_hand_landmark_np_normed:\n            left_hand_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                    image,\n                    left_hand_landmark_np_normed_z,\n                    mp_hands.HAND_CONNECTIONS,\n                    landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        # Face\n        face_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in face_landmark_np_normed:\n            face_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n            image=image,\n            landmark_list=face_landmark_np_normed_z,\n            connections=mp_face_mesh.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles\n            .get_default_face_mesh_tesselation_style())\n        \n        mp_drawing.draw_landmarks(\n            image=image,\n            landmark_list=face_landmark_np_normed_z,\n            connections=mp_face_mesh.FACEMESH_CONTOURS,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles\n            .get_default_face_mesh_contours_style())\n        \n        # Iris data not available. So ignoring the iris visualization.\n        \n        all_images.append(image.astype(np.uint8))\n        all_landmarks_data.append([pose_landmark_np_normed, right_hand_landmark_np_normed_z, left_hand_landmark_np_normed_z, face_landmark_np_normed_z])\n        all_landmarks.append([pose_landmark_np_normed_z, right_hand_landmark_np_normed_z, left_hand_landmark_np_normed_z, face_landmark_np_normed_z])\n    return all_images, all_landmarks_data, all_landmarks","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:50:09.957096Z","iopub.execute_input":"2023-06-03T05:50:09.957921Z","iopub.status.idle":"2023-06-03T05:50:09.978497Z","shell.execute_reply.started":"2023-06-03T05:50:09.957894Z","shell.execute_reply":"2023-06-03T05:50:09.977001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"phrase_idx, phrase, sequence_df = get_random_sequence(0)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:50:09.980338Z","iopub.execute_input":"2023-06-03T05:50:09.980687Z","iopub.status.idle":"2023-06-03T05:50:10.949772Z","shell.execute_reply.started":"2023-06-03T05:50:09.98066Z","shell.execute_reply":"2023-06-03T05:50:10.948317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"{phrase_idx}: {phrase}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:50:10.951574Z","iopub.execute_input":"2023-06-03T05:50:10.952013Z","iopub.status.idle":"2023-06-03T05:50:10.957394Z","shell.execute_reply.started":"2023-06-03T05:50:10.951977Z","shell.execute_reply":"2023-06-03T05:50:10.956424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_images, all_landmarks_data, all_landmarks = get_all_images(sequence_df)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:50:10.963082Z","iopub.execute_input":"2023-06-03T05:50:10.963817Z","iopub.status.idle":"2023-06-03T05:50:17.835965Z","shell.execute_reply.started":"2023-06-03T05:50:10.963768Z","shell.execute_reply":"2023-06-03T05:50:17.834947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_animation(all_images)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:50:17.83787Z","iopub.execute_input":"2023-06-03T05:50:17.838205Z","iopub.status.idle":"2023-06-03T05:50:28.515371Z","shell.execute_reply.started":"2023-06-03T05:50:17.838181Z","shell.execute_reply":"2023-06-03T05:50:28.514692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fill missing values with previous values","metadata":{}},{"cell_type":"code","source":"all_images2, _, _ = get_all_images(sequence_df.fillna(method='ffill'))","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:50:28.516548Z","iopub.execute_input":"2023-06-03T05:50:28.517021Z","iopub.status.idle":"2023-06-03T05:50:34.997015Z","shell.execute_reply.started":"2023-06-03T05:50:28.516998Z","shell.execute_reply":"2023-06-03T05:50:34.995975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_animation(all_images2)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:50:34.998392Z","iopub.execute_input":"2023-06-03T05:50:34.998752Z","iopub.status.idle":"2023-06-03T05:50:45.661924Z","shell.execute_reply.started":"2023-06-03T05:50:34.998722Z","shell.execute_reply":"2023-06-03T05:50:45.660794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3D plot","metadata":{}},{"cell_type":"code","source":"mp_drawing.plot_landmarks(all_landmarks[0][0], mp_pose.POSE_CONNECTIONS, azimuth=0, elevation=0)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:50:45.663123Z","iopub.execute_input":"2023-06-03T05:50:45.663384Z","iopub.status.idle":"2023-06-03T05:50:46.22378Z","shell.execute_reply.started":"2023-06-03T05:50:45.663362Z","shell.execute_reply":"2023-06-03T05:50:46.222466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### There is some shift in the z-axis making the pose bend forwards. I would have expected this to be almost straight.\n### Maybe this needs a rotational transformation. Or this might not be that important. Might want to revisit at a later point.","metadata":{}},{"cell_type":"code","source":"mp_drawing.plot_landmarks(all_landmarks[0][0], mp_pose.POSE_CONNECTIONS, azimuth=90, elevation=0)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T05:50:46.22518Z","iopub.execute_input":"2023-06-03T05:50:46.225515Z","iopub.status.idle":"2023-06-03T05:50:46.77611Z","shell.execute_reply.started":"2023-06-03T05:50:46.225489Z","shell.execute_reply":"2023-06-03T05:50:46.775174Z"},"trusted":true},"execution_count":null,"outputs":[]}]}