{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### This is a notebook I created to explore the dataset and visualize the landmarks to get a feel of the problem.\n### The code uses mediapipe APIs where available.","metadata":{"_uuid":"584f6534-dd38-4808-88b5-366d155c0262","_cell_guid":"1aeadbec-c018-4798-ac78-464df8a6b57b","trusted":true}},{"cell_type":"markdown","source":"# Reference materials for this notebook\n* https://developers.google.com/mediapipe/solutions/guide\n* Pose: https://colab.research.google.com/github/googlesamples/mediapipe/blob/main/examples/pose_landmarker/python/%5BMediaPipe_Python_Tasks%5D_Pose_Landmarker.ipynb\n* Hand: https://colab.research.google.com/github/googlesamples/mediapipe/blob/main/examples/hand_landmarker/python/hand_landmarker.ipynb\n* Face: https://colab.research.google.com/github/googlesamples/mediapipe/blob/main/examples/face_landmarker/python/%5BMediaPipe_Python_Tasks%5D_Face_Landmarker.ipynb\n* mediapipe Python bindings: https://github.com/google/mediapipe/tree/master/mediapipe/python\n* mediapipe samples: https://github.com/googlesamples/mediapipe","metadata":{"_uuid":"8e281cc5-81d7-4a2b-ba0d-ef8d6f757f35","_cell_guid":"231759f5-6e7c-4db7-a648-fb0b57101d76","trusted":true}},{"cell_type":"code","source":"!pip install mediapipe","metadata":{"_uuid":"6f47ed5f-d34a-4fe8-a1e8-2e1c8cce05ac","_cell_guid":"8ed14884-7424-4d60-9cc5-69195ea81b42","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:14.400008Z","iopub.execute_input":"2023-07-21T00:27:14.40061Z","iopub.status.idle":"2023-07-21T00:27:27.794194Z","shell.execute_reply.started":"2023-07-21T00:27:14.40057Z","shell.execute_reply":"2023-07-21T00:27:27.793047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nfrom mediapipe.framework.formats import landmark_pb2\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nimport mediapipe\nfrom mediapipe.framework.formats import landmark_pb2\nfrom tqdm.notebook import tqdm\nimport matplotlib\nmatplotlib.rcParams['animation.embed_limit'] = 2**128\nmatplotlib.rcParams['savefig.pad_inches'] = 0","metadata":{"_uuid":"a731172d-0929-4fad-be45-9ae2cb480fd4","_cell_guid":"ff3fa051-51ff-44a6-9623-8ed4ad0abffc","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:27.797057Z","iopub.execute_input":"2023-07-21T00:27:27.797464Z","iopub.status.idle":"2023-07-21T00:27:37.833993Z","shell.execute_reply.started":"2023-07-21T00:27:27.797421Z","shell.execute_reply":"2023-07-21T00:27:37.832875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_dir = Path('/kaggle/input/asl-fingerspelling/')","metadata":{"_uuid":"dfaea00c-bc16-4ab4-99f1-973868f9b4b5","_cell_guid":"ac4260c5-03aa-4fb7-8525-65a4008b2b8f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:37.835435Z","iopub.execute_input":"2023-07-21T00:27:37.836185Z","iopub.status.idle":"2023-07-21T00:27:37.841533Z","shell.execute_reply.started":"2023-07-21T00:27:37.836151Z","shell.execute_reply":"2023-07-21T00:27:37.840338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visual Helpers","metadata":{}},{"cell_type":"code","source":"from matplotlib import animation, rc\nrc('animation', html='jshtml')\n\ndef create_animation(images):\n    fig = plt.figure(figsize=(6, 9))\n    ax = plt.Axes(fig, [0., 0., 1., 1.])\n    ax.set_axis_off()\n    fig.add_axes(ax)\n    im=ax.imshow(images[0], cmap=\"gray\")\n    plt.close(fig)\n    \n    def animate_func(i):\n        im.set_array(images[i])\n        return [im]\n\n    return animation.FuncAnimation(fig, animate_func, frames=len(images), interval=1000/10)","metadata":{"_uuid":"af38dad7-a9ff-4152-895e-2faac0f9930c","_cell_guid":"6341efb7-78d9-48f4-a4c4-e351ea7aa264","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:37.843105Z","iopub.execute_input":"2023-07-21T00:27:37.843471Z","iopub.status.idle":"2023-07-21T00:27:37.868733Z","shell.execute_reply.started":"2023-07-21T00:27:37.84344Z","shell.execute_reply":"2023-07-21T00:27:37.867468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_random_sequence(phrase_idx=None):\n    if phrase_idx == None:\n        phrase_idx = np.random.randint(len(train_df))\n\n    selected_row = train_df.iloc[phrase_idx]\n    file_id = selected_row['file_id']\n    sequence_id = selected_row['sequence_id']\n    phrase = selected_row['phrase']\n    \n    parquet_file = pq.ParquetFile(base_dir / 'train_landmarks' / f\"{str(file_id)}.parquet\")\n    \n    dataset = pq.read_table(\n        base_dir / 'train_landmarks' / f\"{str(file_id)}.parquet\",\n        filters=[\n            [('sequence_id', '=', sequence_id)],\n        ]\n    )\n    \n    sequence_df = dataset.to_pandas()\n    return phrase_idx, phrase, sequence_df","metadata":{"_uuid":"af38dad7-a9ff-4152-895e-2faac0f9930c","_cell_guid":"6341efb7-78d9-48f4-a4c4-e351ea7aa264","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:37.873047Z","iopub.execute_input":"2023-07-21T00:27:37.874217Z","iopub.status.idle":"2023-07-21T00:27:37.882155Z","shell.execute_reply.started":"2023-07-21T00:27:37.874181Z","shell.execute_reply":"2023-07-21T00:27:37.881051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hands(seq_df):\n    images = []\n    all_hand_landmarks = []\n    for seq_idx in range(len(seq_df)):\n        x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_right_hand.*\").values\n        y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_right_hand.*\").values\n        z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_right_hand.*\").values\n\n        right_hand_image = np.zeros((900, 600, 3))\n\n        right_hand_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for x, y, z in zip(x_pose, y_pose, z_pose):\n            right_hand_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n                right_hand_image,\n                right_hand_landmarks,\n                mp_hands.HAND_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_left_hand.*\").values\n        y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_left_hand.*\").values\n        z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_left_hand.*\").values\n        \n        left_hand_image = np.zeros((900, 600, 3))\n        \n        left_hand_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for x, y, z in zip(x_pose, y_pose, z_pose):\n            left_hand_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n                left_hand_image,\n                left_hand_landmarks,\n                mp_hands.HAND_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        images.append([right_hand_image.astype(np.uint8), left_hand_image.astype(np.uint8)])\n        all_hand_landmarks.append([right_hand_landmarks, left_hand_landmarks])\n    return images, all_hand_landmarks","metadata":{"_uuid":"af38dad7-a9ff-4152-895e-2faac0f9930c","_cell_guid":"6341efb7-78d9-48f4-a4c4-e351ea7aa264","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:37.883536Z","iopub.execute_input":"2023-07-21T00:27:37.883875Z","iopub.status.idle":"2023-07-21T00:27:37.899026Z","shell.execute_reply.started":"2023-07-21T00:27:37.883847Z","shell.execute_reply":"2023-07-21T00:27:37.897755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_face(seq_df):\n    images = []\n    all_face_landmarks = []\n    for seq_idx in range(len(seq_df)):\n        x_face = seq_df.iloc[seq_idx].filter(regex=\"x_face.*\").values\n        y_face = seq_df.iloc[seq_idx].filter(regex=\"y_face.*\").values\n        z_face = seq_df.iloc[seq_idx].filter(regex=\"z_face.*\").values\n\n        annotated_image = np.zeros((900, 600, 3))\n\n        face_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for x, y, z in zip(x_face, y_face, z_face):\n            face_landmarks.landmark.add(x=x, y=y, z=z)\n\n        mp_drawing.draw_landmarks(\n          image=annotated_image,\n          landmark_list=face_landmarks,\n          connections=mp_face_mesh.FACEMESH_TESSELATION,\n          landmark_drawing_spec=None,\n          connection_drawing_spec=mp_drawing_styles\n          .get_default_face_mesh_tesselation_style())\n        mp_drawing.draw_landmarks(\n          image=annotated_image,\n          landmark_list=face_landmarks,\n          connections=mp_face_mesh.FACEMESH_CONTOURS,\n          landmark_drawing_spec=None,\n          connection_drawing_spec=mp_drawing_styles\n          .get_default_face_mesh_contours_style())\n\n        images.append(annotated_image.astype(np.uint8))\n        all_face_landmarks.append(face_landmarks)\n    return images, all_face_landmarks","metadata":{"_uuid":"af38dad7-a9ff-4152-895e-2faac0f9930c","_cell_guid":"6341efb7-78d9-48f4-a4c4-e351ea7aa264","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:37.900724Z","iopub.execute_input":"2023-07-21T00:27:37.901193Z","iopub.status.idle":"2023-07-21T00:27:37.917427Z","shell.execute_reply.started":"2023-07-21T00:27:37.901152Z","shell.execute_reply":"2023-07-21T00:27:37.916398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pose(seq_df):\n    images = []\n    all_pose_landmarks = []\n    for seq_idx in range(len(seq_df)):\n        x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_pose.*\").values\n        y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_pose.*\").values\n        z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_pose.*\").values\n\n        annotated_image = np.zeros((900, 600, 3))\n        \n        data_points = []\n        for x, y, z in zip(x_pose, y_pose, z_pose):\n            data_points.append(np.array([x, y, z]))\n\n        pose_landmarks = landmark_pb2.NormalizedLandmarkList()\n        for row in data_points:\n            pose_landmarks.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                annotated_image,\n                pose_landmarks,\n                mp_pose.POSE_CONNECTIONS,\n                landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())\n        images.append(annotated_image.astype(np.uint8))\n        all_pose_landmarks.append(pose_landmarks)\n    return images, all_pose_landmarks","metadata":{"_uuid":"a7025ef8-5242-4453-aba6-2827fb905fd1","_cell_guid":"37896e12-9258-4235-8840-814e88afc9a2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:37.918918Z","iopub.execute_input":"2023-07-21T00:27:37.91938Z","iopub.status.idle":"2023-07-21T00:27:37.93915Z","shell.execute_reply.started":"2023-07-21T00:27:37.919348Z","shell.execute_reply":"2023-07-21T00:27:37.93806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_landmark_to_npy(landmarklist):\n    return np.array([np.array([landmark.x, landmark.y, landmark.z]) for landmark in landmarklist.landmark])","metadata":{"_uuid":"b66e224e-537a-41e8-9a3f-531022c1e9fd","_cell_guid":"d3f1c0c3-0c9b-4074-94c5-472bfd881924","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:40:43.623836Z","iopub.execute_input":"2023-07-21T00:40:43.624933Z","iopub.status.idle":"2023-07-21T00:40:43.630537Z","shell.execute_reply.started":"2023-07-21T00:40:43.624882Z","shell.execute_reply":"2023-07-21T00:40:43.629236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_all_images(seq_df):\n    pose_images, pose_landmarks = get_pose(sequence_df)\n    hand_images, hand_landmarks = get_hands(sequence_df)\n    face_images, face_landmarks = get_face(sequence_df)\n    \n    all_images = []\n    all_landmarks_data = []\n    all_landmarks = []\n    for seq_idx in tqdm(range(len(pose_landmarks))):\n        pose_landmark_np = convert_landmark_to_npy(pose_landmarks[seq_idx])\n        right_hand_landmark_np = convert_landmark_to_npy(hand_landmarks[seq_idx][0])\n        left_hand_landmark_np = convert_landmark_to_npy(hand_landmarks[seq_idx][1])\n        face_landmark_np = convert_landmark_to_npy(face_landmarks[seq_idx])\n        \n        # Pool all landmarks together to find min and max coordinates\n        pooled_landmarks = np.vstack((pose_landmark_np, right_hand_landmark_np, left_hand_landmark_np, face_landmark_np))\n        pooled_min = np.nanmin(pooled_landmarks, axis=0)\n        pooled_max = np.nanmax(pooled_landmarks, axis=0)\n        \n        # Use the max of x and y scaling to proportionally scale the image. We don't need to scale z for 2D image\n        # There is an elegant way to achieve the same result as below with Matrix transformations. Might re-do this if there is interest.\n        scaling_factor = np.nanmax(pooled_max[:2])\n        pooled_scaled_min = np.nanmin(pooled_landmarks / scaling_factor, axis=0)\n\n        pose_landmark_np_normed = (pose_landmark_np / scaling_factor) - pooled_scaled_min\n        \n        # Center the image around shoulder and hips. Makes for a better visualization\n        x_shift = ((1-(pose_landmark_np_normed[23]+pose_landmark_np_normed[24]))/2)[0]\n        axis_shift = np.array([x_shift, 0, 0])\n        \n        pose_landmark_np_normed = pose_landmark_np_normed + axis_shift\n        right_hand_landmark_np_normed = (right_hand_landmark_np / scaling_factor) - pooled_scaled_min + axis_shift\n        left_hand_landmark_np_normed = (left_hand_landmark_np / scaling_factor) - pooled_scaled_min  + axis_shift\n        face_landmark_np_normed = (face_landmark_np / scaling_factor) - pooled_scaled_min + axis_shift\n        \n        # Now that we have scaled and shifted the landmarks to fit into a [0, 1] range, we can start plotting them using mediapipe APIs\n        # BG image with zeros\n        image = np.zeros((900, 600, 3))\n        \n        # Pose\n        pose_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in pose_landmark_np_normed:\n            pose_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                    image,\n                    pose_landmark_np_normed_z,\n                    mp_pose.POSE_CONNECTIONS,\n                    landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())\n\n        # Right hand\n        right_hand_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in right_hand_landmark_np_normed:\n            right_hand_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                    image,\n                    right_hand_landmark_np_normed_z,\n                    mp_hands.HAND_CONNECTIONS,\n                    landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        # Left hand\n        left_hand_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in left_hand_landmark_np_normed:\n            left_hand_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n                    image,\n                    left_hand_landmark_np_normed_z,\n                    mp_hands.HAND_CONNECTIONS,\n                    landmark_drawing_spec=mp_drawing_styles.get_default_hand_landmarks_style())\n        \n        # Face\n        face_landmark_np_normed_z = landmark_pb2.LandmarkList()\n        for row in face_landmark_np_normed:\n            face_landmark_np_normed_z.landmark.add(x=row[0], y=row[1], z=row[2])\n\n        mp_drawing.draw_landmarks(\n            image=image,\n            landmark_list=face_landmark_np_normed_z,\n            connections=mp_face_mesh.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles\n            .get_default_face_mesh_tesselation_style())\n        \n        mp_drawing.draw_landmarks(\n            image=image,\n            landmark_list=face_landmark_np_normed_z,\n            connections=mp_face_mesh.FACEMESH_CONTOURS,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles\n            .get_default_face_mesh_contours_style())\n        \n        # Iris data not available. So ignoring the iris visualization.\n        \n        all_images.append(image.astype(np.uint8))\n        all_landmarks_data.append([pose_landmark_np_normed, right_hand_landmark_np_normed_z, left_hand_landmark_np_normed_z, face_landmark_np_normed_z])\n        all_landmarks.append([pose_landmark_np_normed_z, right_hand_landmark_np_normed_z, left_hand_landmark_np_normed_z, face_landmark_np_normed_z])\n    return all_images, all_landmarks_data, all_landmarks","metadata":{"_uuid":"90cdd716-4fee-44a4-b4bc-17aa3b232626","_cell_guid":"1c73b344-e942-4b72-9acc-a6068665fe98","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:39:36.079641Z","iopub.execute_input":"2023-07-21T00:39:36.080306Z","iopub.status.idle":"2023-07-21T00:39:36.104056Z","shell.execute_reply.started":"2023-07-21T00:39:36.080261Z","shell.execute_reply":"2023-07-21T00:39:36.103005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mediapipe helpers","metadata":{"_uuid":"2e7a8622-b208-43bb-8e59-ee63661de847","_cell_guid":"da9bb7ec-823c-4ced-b521-8121acd39c88","trusted":true}},{"cell_type":"code","source":"mp_pose = mediapipe.solutions.pose\nmp_hands = mediapipe.solutions.hands\nmp_face_mesh = mediapipe.solutions.face_mesh\nmp_drawing = mediapipe.solutions.drawing_utils \nmp_drawing_styles = mediapipe.solutions.drawing_styles","metadata":{"_uuid":"79d4e4d8-5101-49ce-b4fe-3c4c84f5f94c","_cell_guid":"cf0aed08-e946-44ac-94af-7a7d52fca6af","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:37.941062Z","iopub.execute_input":"2023-07-21T00:27:37.941472Z","iopub.status.idle":"2023-07-21T00:27:37.956815Z","shell.execute_reply.started":"2023-07-21T00:27:37.94144Z","shell.execute_reply":"2023-07-21T00:27:37.955769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(base_dir / 'train.csv')","metadata":{"_uuid":"a6bae4a5-dd78-42d5-aa2d-6406b438995a","_cell_guid":"5d8499fd-0551-46d4-ac0d-a5bedbdb026d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:37.958664Z","iopub.execute_input":"2023-07-21T00:27:37.959117Z","iopub.status.idle":"2023-07-21T00:27:38.145383Z","shell.execute_reply.started":"2023-07-21T00:27:37.959073Z","shell.execute_reply":"2023-07-21T00:27:38.144165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"_uuid":"d39eda9f-fc83-40b8-b911-b8580f8dc9b5","_cell_guid":"1e6cf828-a0d9-40d6-8e07-a7b8bf34abe9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:38.147009Z","iopub.execute_input":"2023-07-21T00:27:38.14739Z","iopub.status.idle":"2023-07-21T00:27:38.182698Z","shell.execute_reply.started":"2023-07-21T00:27:38.147359Z","shell.execute_reply":"2023-07-21T00:27:38.180843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of file_ids","metadata":{}},{"cell_type":"code","source":"file_ids = train_df.file_id.unique()\nfile_ids.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:27:38.184447Z","iopub.execute_input":"2023-07-21T00:27:38.184786Z","iopub.status.idle":"2023-07-21T00:27:38.197617Z","shell.execute_reply.started":"2023-07-21T00:27:38.184759Z","shell.execute_reply":"2023-07-21T00:27:38.196263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Shape of each file_id","metadata":{}},{"cell_type":"code","source":"train_df.loc[train_df.file_id == file_ids[0]]","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:27:38.199535Z","iopub.execute_input":"2023-07-21T00:27:38.200659Z","iopub.status.idle":"2023-07-21T00:27:38.224402Z","shell.execute_reply.started":"2023-07-21T00:27:38.200624Z","shell.execute_reply":"2023-07-21T00:27:38.223266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[train_df.file_id == file_ids[1]]","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:27:38.230279Z","iopub.execute_input":"2023-07-21T00:27:38.230679Z","iopub.status.idle":"2023-07-21T00:27:38.246721Z","shell.execute_reply.started":"2023-07-21T00:27:38.230646Z","shell.execute_reply":"2023-07-21T00:27:38.245708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_ids.shape[0]*1000 == train_df.shape[0] - 2","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:27:38.248034Z","iopub.execute_input":"2023-07-21T00:27:38.248572Z","iopub.status.idle":"2023-07-21T00:27:38.254279Z","shell.execute_reply.started":"2023-07-21T00:27:38.248541Z","shell.execute_reply":"2023-07-21T00:27:38.253498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[train_df.file_id == file_ids[-1]]","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:27:38.255991Z","iopub.execute_input":"2023-07-21T00:27:38.256312Z","iopub.status.idle":"2023-07-21T00:27:38.28037Z","shell.execute_reply.started":"2023-07-21T00:27:38.256286Z","shell.execute_reply":"2023-07-21T00:27:38.279335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx in file_ids:\n    print(f'{idx} get shape : {train_df.loc[train_df.file_id == idx].shape}')","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:27:38.281513Z","iopub.execute_input":"2023-07-21T00:27:38.282234Z","iopub.status.idle":"2023-07-21T00:27:38.327406Z","shell.execute_reply.started":"2023-07-21T00:27:38.282201Z","shell.execute_reply":"2023-07-21T00:27:38.326053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"phrase_idx = 3\nselected_row = train_df.iloc[phrase_idx]\nfile_id = selected_row['file_id']\nsequence_id = selected_row['sequence_id']\nphrase = selected_row['phrase']\nprint(f\"Phrase: {phrase}\")","metadata":{"_uuid":"826d86fc-84a7-49e8-8cad-29d24308f82a","_cell_guid":"7ccb9a12-acba-44b3-ac08-60b3bdbbfccb","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:38.329002Z","iopub.execute_input":"2023-07-21T00:27:38.329438Z","iopub.status.idle":"2023-07-21T00:27:38.336583Z","shell.execute_reply.started":"2023-07-21T00:27:38.329407Z","shell.execute_reply":"2023-07-21T00:27:38.335218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_id","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:27:38.338125Z","iopub.execute_input":"2023-07-21T00:27:38.338494Z","iopub.status.idle":"2023-07-21T00:27:38.353004Z","shell.execute_reply.started":"2023-07-21T00:27:38.338462Z","shell.execute_reply":"2023-07-21T00:27:38.352153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquet_file = pq.ParquetFile(base_dir / 'train_landmarks' / f\"{file_id}.parquet\")\nparquet_file","metadata":{"_uuid":"6439f21d-d47b-415f-82b7-cb4cb31f75c4","_cell_guid":"7e6bdd2f-9ac9-4446-9497-bcf9d4997774","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:38.354296Z","iopub.execute_input":"2023-07-21T00:27:38.354842Z","iopub.status.idle":"2023-07-21T00:27:38.420419Z","shell.execute_reply.started":"2023-07-21T00:27:38.354812Z","shell.execute_reply":"2023-07-21T00:27:38.419113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read only the relevant rows for this sequence ID. Might come handy when creating a data loader for model training","metadata":{"_uuid":"8ee76a7b-20d3-4620-85d0-312b895dcc1c","_cell_guid":"c7c12f75-1e25-4577-9849-dc31dfea727a","trusted":true}},{"cell_type":"code","source":"dataset = pq.read_table(\n    base_dir / 'train_landmarks' / f\"{file_id}.parquet\",\n    filters=[\n        [('sequence_id', '=', sequence_id)],\n    ]\n)","metadata":{"_uuid":"e3dc469b-d707-4368-a975-c2abe81f11a3","_cell_guid":"1e4cd07f-2e52-4891-9b02-ffca6334c49f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:38.422293Z","iopub.execute_input":"2023-07-21T00:27:38.422912Z","iopub.status.idle":"2023-07-21T00:27:41.935643Z","shell.execute_reply.started":"2023-07-21T00:27:38.422874Z","shell.execute_reply":"2023-07-21T00:27:41.934632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_df = dataset.to_pandas()","metadata":{"_uuid":"67ddc30a-2f3c-4e9d-9742-d4aa575d6e86","_cell_guid":"f77953e6-482e-4bd5-96c0-15a9a766184c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:41.936665Z","iopub.execute_input":"2023-07-21T00:27:41.937018Z","iopub.status.idle":"2023-07-21T00:27:41.99288Z","shell.execute_reply.started":"2023-07-21T00:27:41.936982Z","shell.execute_reply":"2023-07-21T00:27:41.991989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_df.head()","metadata":{"_uuid":"98c0ae8a-33fb-4913-bf43-91a834bd2490","_cell_guid":"94d743c7-60c4-4ac1-b446-7ddf77c355b4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:41.994714Z","iopub.execute_input":"2023-07-21T00:27:41.995274Z","iopub.status.idle":"2023-07-21T00:27:42.030014Z","shell.execute_reply.started":"2023-07-21T00:27:41.995241Z","shell.execute_reply":"2023-07-21T00:27:42.028698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_df.columns","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:27:42.031487Z","iopub.execute_input":"2023-07-21T00:27:42.032093Z","iopub.status.idle":"2023-07-21T00:27:42.039228Z","shell.execute_reply.started":"2023-07-21T00:27:42.032063Z","shell.execute_reply":"2023-07-21T00:27:42.038039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hand_images, hand_landmarks = get_hands(sequence_df)","metadata":{"_uuid":"3426b52d-4374-45ff-b501-055275d7b3b3","_cell_guid":"12f1e238-dc54-4d5a-a9da-4b9c8a2b0cbe","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:42.040838Z","iopub.execute_input":"2023-07-21T00:27:42.042172Z","iopub.status.idle":"2023-07-21T00:27:45.379067Z","shell.execute_reply.started":"2023-07-21T00:27:42.04213Z","shell.execute_reply":"2023-07-21T00:27:45.377855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Right hand","metadata":{"_uuid":"3d252402-1ce9-454a-bbfc-3d404345b701","_cell_guid":"b8959556-8bd6-4ab7-9efd-c432bd791a22","trusted":true}},{"cell_type":"code","source":"create_animation(np.array(hand_images)[:, 0])","metadata":{"_uuid":"8d6ff5c5-83ee-4eed-b4e5-080b5c0ea54d","_cell_guid":"d8314b34-0345-478b-934b-b8856bbe1843","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:45.380595Z","iopub.execute_input":"2023-07-21T00:27:45.381348Z","iopub.status.idle":"2023-07-21T00:27:56.630515Z","shell.execute_reply.started":"2023-07-21T00:27:45.381316Z","shell.execute_reply":"2023-07-21T00:27:56.629082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Left hand","metadata":{"_uuid":"154ad7e6-cca4-47e3-a8cb-74708c6fb5fd","_cell_guid":"406c1cb9-0dbe-4a9a-8874-1f186e9ae46e","trusted":true}},{"cell_type":"code","source":"create_animation(np.array(hand_images)[:, 1])","metadata":{"_uuid":"37c532f6-ffc6-4e10-abb3-ca50dbff7f31","_cell_guid":"4f23011f-adde-42b9-97de-4454c2c61aea","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:27:56.632064Z","iopub.execute_input":"2023-07-21T00:27:56.632481Z","iopub.status.idle":"2023-07-21T00:28:07.553517Z","shell.execute_reply.started":"2023-07-21T00:27:56.632445Z","shell.execute_reply":"2023-07-21T00:28:07.552229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"face_images, face_landmarks = get_face(sequence_df)","metadata":{"_uuid":"45717263-ca76-4619-90c1-6e6b68d98787","_cell_guid":"008e4951-03cc-42b7-8538-2aaf54ab6ec1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:28:07.555063Z","iopub.execute_input":"2023-07-21T00:28:07.555428Z","iopub.status.idle":"2023-07-21T00:28:11.02965Z","shell.execute_reply.started":"2023-07-21T00:28:07.555397Z","shell.execute_reply":"2023-07-21T00:28:11.028361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Face","metadata":{"_uuid":"4b37edff-0dc7-4b82-9002-8829c4ba2a48","_cell_guid":"446f35a3-8232-4a19-931d-680c2ef7d229","trusted":true}},{"cell_type":"code","source":"create_animation(face_images)","metadata":{"_uuid":"e715ac7c-d095-47d9-b808-7ed18222d4c3","_cell_guid":"191aa101-e2e1-49e1-a3ba-418f718e80c2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:28:11.031257Z","iopub.execute_input":"2023-07-21T00:28:11.032564Z","iopub.status.idle":"2023-07-21T00:28:22.907846Z","shell.execute_reply.started":"2023-07-21T00:28:11.032522Z","shell.execute_reply":"2023-07-21T00:28:22.906386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pose_images, pose_landmarks = get_pose(sequence_df)","metadata":{"_uuid":"dda313a8-c636-490b-a97e-e4a78a242219","_cell_guid":"e984f6f7-af13-4f08-8198-a5b74ecb7cee","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:28:22.909375Z","iopub.execute_input":"2023-07-21T00:28:22.909784Z","iopub.status.idle":"2023-07-21T00:28:24.571442Z","shell.execute_reply.started":"2023-07-21T00:28:22.909755Z","shell.execute_reply":"2023-07-21T00:28:24.57026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pose","metadata":{"_uuid":"949ad519-88fa-4de2-809a-8b8de7c6cb48","_cell_guid":"fdaae414-5c8c-43eb-b3ec-11ef3c6978f7","execution":{"iopub.status.busy":"2023-06-03T05:34:22.499227Z","iopub.execute_input":"2023-06-03T05:34:22.499555Z","iopub.status.idle":"2023-06-03T05:34:22.506034Z","shell.execute_reply.started":"2023-06-03T05:34:22.499528Z","shell.execute_reply":"2023-06-03T05:34:22.504308Z"},"trusted":true}},{"cell_type":"markdown","source":"### Notice how the pose does not fit in the image. This is because the landmarks are not normalized to [0, 1].\n### We will fix this next","metadata":{"_uuid":"2d1dc27a-dddf-41e4-b67e-c1c615684666","_cell_guid":"8b37a7fa-dd83-4001-9204-142706b36c79","trusted":true}},{"cell_type":"code","source":"create_animation(pose_images)","metadata":{"_uuid":"f47551a3-a63a-40f1-878d-ab4a14f29bb8","_cell_guid":"accced36-7fd4-400b-ae74-a24c553fe254","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:28:24.573289Z","iopub.execute_input":"2023-07-21T00:28:24.573725Z","iopub.status.idle":"2023-07-21T00:28:35.721002Z","shell.execute_reply.started":"2023-07-21T00:28:24.573687Z","shell.execute_reply":"2023-07-21T00:28:35.719834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"phrase_idx, phrase, sequence_df = get_random_sequence(0)","metadata":{"_uuid":"6e66865c-9dff-4cee-958e-9f38550a3d22","_cell_guid":"6c28c127-10a4-43cd-b78a-584450141c0e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:39:38.949814Z","iopub.execute_input":"2023-07-21T00:39:38.95025Z","iopub.status.idle":"2023-07-21T00:39:40.551044Z","shell.execute_reply.started":"2023-07-21T00:39:38.950216Z","shell.execute_reply":"2023-07-21T00:39:40.550022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"{phrase_idx}: {phrase}\")","metadata":{"_uuid":"8554d077-dc6c-4c0c-8e0b-d7814af6ae27","_cell_guid":"b823947d-5bfe-4313-9fb7-50178d7e62aa","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:39:56.942829Z","iopub.execute_input":"2023-07-21T00:39:56.943292Z","iopub.status.idle":"2023-07-21T00:39:56.950062Z","shell.execute_reply.started":"2023-07-21T00:39:56.943261Z","shell.execute_reply":"2023-07-21T00:39:56.948716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_images, all_landmarks_data, all_landmarks = get_all_images(sequence_df)","metadata":{"_uuid":"c051c4cc-d302-40b9-908e-cba683302e57","_cell_guid":"d197ab78-fc4b-4dcd-bd38-a04bf326ae4c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:42:51.913977Z","iopub.execute_input":"2023-07-21T00:42:51.914458Z","iopub.status.idle":"2023-07-21T00:43:05.25275Z","shell.execute_reply.started":"2023-07-21T00:42:51.914426Z","shell.execute_reply":"2023-07-21T00:43:05.251858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_images)","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:46:33.761203Z","iopub.execute_input":"2023-07-21T00:46:33.761658Z","iopub.status.idle":"2023-07-21T00:46:33.769527Z","shell.execute_reply.started":"2023-07-21T00:46:33.761622Z","shell.execute_reply":"2023-07-21T00:46:33.768294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_animation(all_images)","metadata":{"_uuid":"715572ff-ddca-4131-ad68-229d1ba896c0","_cell_guid":"7b1d695e-4e14-4cce-8f57-333f782e0987","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:43:36.25Z","iopub.execute_input":"2023-07-21T00:43:36.250534Z","iopub.status.idle":"2023-07-21T00:43:50.630085Z","shell.execute_reply.started":"2023-07-21T00:43:36.25049Z","shell.execute_reply":"2023-07-21T00:43:50.628402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fill missing values with previous values","metadata":{"_uuid":"1f53bbab-2a3d-4c18-bf32-3c6921b80a4c","_cell_guid":"da43d9b6-e23e-46ae-bbf7-f37981dc3140","trusted":true}},{"cell_type":"code","source":"all_images2, _, _ = get_all_images(sequence_df.fillna(method='ffill'))","metadata":{"_uuid":"df140d28-785e-4315-a9e0-49f3d17b4c37","_cell_guid":"47c09a46-35e0-4153-9bbc-ba8642f99f6e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:45:06.264648Z","iopub.execute_input":"2023-07-21T00:45:06.265148Z","iopub.status.idle":"2023-07-21T00:45:18.70941Z","shell.execute_reply.started":"2023-07-21T00:45:06.265109Z","shell.execute_reply":"2023-07-21T00:45:18.708169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_images2)","metadata":{"execution":{"iopub.status.busy":"2023-07-21T00:46:51.142887Z","iopub.execute_input":"2023-07-21T00:46:51.143841Z","iopub.status.idle":"2023-07-21T00:46:51.150866Z","shell.execute_reply.started":"2023-07-21T00:46:51.143806Z","shell.execute_reply":"2023-07-21T00:46:51.149755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_animation(all_images2)","metadata":{"_uuid":"bc14e340-2c48-4575-95ee-c87fd50fb102","_cell_guid":"db357e66-8e77-4881-842b-fdd22af8399c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:45:38.455361Z","iopub.execute_input":"2023-07-21T00:45:38.455813Z","iopub.status.idle":"2023-07-21T00:45:52.781474Z","shell.execute_reply.started":"2023-07-21T00:45:38.455779Z","shell.execute_reply":"2023-07-21T00:45:52.780162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3D plot","metadata":{"_uuid":"9c14ac28-0cbf-457d-9a44-acc7b4f36c91","_cell_guid":"ad585515-fd60-4259-ae9a-63b0eb2cb378","trusted":true}},{"cell_type":"code","source":"mp_drawing.plot_landmarks(all_landmarks[0][0], mp_pose.POSE_CONNECTIONS, azimuth=0, elevation=0)","metadata":{"_uuid":"2212cd67-36eb-42b3-a813-789bb8b7e5c7","_cell_guid":"9a30f7a1-98ee-46b5-a9fd-d39b0efbb225","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:47:13.913723Z","iopub.execute_input":"2023-07-21T00:47:13.914158Z","iopub.status.idle":"2023-07-21T00:47:14.686087Z","shell.execute_reply.started":"2023-07-21T00:47:13.914124Z","shell.execute_reply":"2023-07-21T00:47:14.684929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### There is some shift in the z-axis making the pose bend forwards. I would have expected this to be almost straight.\n### Maybe this needs a rotational transformation. Or this might not be that important. Might want to revisit at a later point.","metadata":{"_uuid":"4f6ce70f-c130-4e60-861b-4ef6423f9377","_cell_guid":"8ef65fa0-5a2c-4f1d-8bed-0871e6779ef2","trusted":true}},{"cell_type":"code","source":"mp_drawing.plot_landmarks(all_landmarks[0][0], mp_pose.POSE_CONNECTIONS, azimuth=90, elevation=0)","metadata":{"_uuid":"a6a4445d-2d09-416f-88ce-e944c1219435","_cell_guid":"204feefc-2745-4dcc-a200-f26c2c7a127c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-21T00:47:23.598448Z","iopub.execute_input":"2023-07-21T00:47:23.598897Z","iopub.status.idle":"2023-07-21T00:47:24.313318Z","shell.execute_reply.started":"2023-07-21T00:47:23.598852Z","shell.execute_reply":"2023-07-21T00:47:24.311853Z"},"trusted":true},"execution_count":null,"outputs":[]}]}