{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q mediapipe","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:16:06.297861Z","iopub.execute_input":"2023-08-06T15:16:06.298557Z","iopub.status.idle":"2023-08-06T15:16:17.998031Z","shell.execute_reply.started":"2023-08-06T15:16:06.298506Z","shell.execute_reply":"2023-08-06T15:16:17.997322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport mediapipe as mp\nimport tensorflow as tf\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:16:26.284255Z","iopub.execute_input":"2023-08-06T15:16:26.28458Z","iopub.status.idle":"2023-08-06T15:16:34.752462Z","shell.execute_reply.started":"2023-08-06T15:16:26.284557Z","shell.execute_reply":"2023-08-06T15:16:34.751448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"in_dir = \"/kaggle/input/asl-fingerspelling/\"\ntrain = pd.read_csv(f\"{in_dir}train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:17:41.773779Z","iopub.execute_input":"2023-08-06T15:17:41.774407Z","iopub.status.idle":"2023-08-06T15:17:41.895024Z","shell.execute_reply.started":"2023-08-06T15:17:41.774379Z","shell.execute_reply":"2023-08-06T15:17:41.893881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:17:54.400667Z","iopub.execute_input":"2023-08-06T15:17:54.402082Z","iopub.status.idle":"2023-08-06T15:17:54.42035Z","shell.execute_reply.started":"2023-08-06T15:17:54.402039Z","shell.execute_reply":"2023-08-06T15:17:54.418961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lme = pd.read_parquet(f\"{in_dir}/{train['path'].iloc[0]}\")\nlme.head()\nlen(lme.columns)","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:34:17.254328Z","iopub.execute_input":"2023-08-06T15:34:17.25471Z","iopub.status.idle":"2023-08-06T15:34:18.587619Z","shell.execute_reply.started":"2023-08-06T15:34:17.254684Z","shell.execute_reply":"2023-08-06T15:34:18.586693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lme.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-06T15:53:26.160518Z","iopub.execute_input":"2023-08-06T15:53:26.160863Z","iopub.status.idle":"2023-08-06T15:53:26.182908Z","shell.execute_reply.started":"2023-08-06T15:53:26.160838Z","shell.execute_reply":"2023-08-06T15:53:26.181573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c = 0\nd = 0\ne = 0\nf = 0\n\nfor i in lme.columns:\n    if \"face\" in i:\n        c += 1\n    elif \"pose\" in i:\n        e += 1\n    elif \"left_hand\" in i:\n        d += 1\n    elif \"right_hand\" in i:\n        f += 1\nc, d, e, f","metadata":{"execution":{"iopub.status.busy":"2023-08-06T17:55:12.377819Z","iopub.execute_input":"2023-08-06T17:55:12.378204Z","iopub.status.idle":"2023-08-06T17:55:12.389614Z","shell.execute_reply.started":"2023-08-06T17:55:12.378167Z","shell.execute_reply":"2023-08-06T17:55:12.387904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lme[lme.columns[1:470]]","metadata":{"execution":{"iopub.status.busy":"2023-08-06T17:42:22.013049Z","iopub.execute_input":"2023-08-06T17:42:22.013815Z","iopub.status.idle":"2023-08-06T17:42:22.189861Z","shell.execute_reply.started":"2023-08-06T17:42:22.013777Z","shell.execute_reply":"2023-08-06T17:42:22.188933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nfrom mediapipe.framework.formats import landmark_pb2\nmp_drawing = mp.solutions.drawing_utils\nmp_drawing_styles = mp.solutions.drawing_styles\nmp_pose = mp.solutions.pose\nmp_holistic = mp.solutions.holistic","metadata":{"execution":{"iopub.status.busy":"2023-08-06T17:10:02.919827Z","iopub.execute_input":"2023-08-06T17:10:02.920217Z","iopub.status.idle":"2023-08-06T17:10:02.928275Z","shell.execute_reply.started":"2023-08-06T17:10:02.920191Z","shell.execute_reply":"2023-08-06T17:10:02.927074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_landmarks(landmarks,image,show_pose=True,show_face_contour=True,show_face_tesselation=True,show_left_hand=True,show_right_hand=True):\n    annotated_image = image.copy()\n    results = landmarks\n    if show_face_tesselation:\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.face_landmarks,\n            mp_holistic.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles\n            .get_default_face_mesh_tesselation_style())\n    if show_face_contour:\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.face_landmarks,\n            mp_holistic.FACEMESH_CONTOURS,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles\n            .get_default_face_mesh_contours_style())\n    if show_pose:\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.pose_landmarks,\n            mp_holistic.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.\n            get_default_pose_landmarks_style())\n    if show_left_hand:\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.left_hand_landmarks,\n            mp_holistic.HAND_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles\n            .get_default_hand_landmarks_style())\n    if show_right_hand:\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.right_hand_landmarks,\n            mp_holistic.HAND_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles\n            .get_default_hand_landmarks_style())\n    return annotated_image","metadata":{"execution":{"iopub.status.busy":"2023-08-06T17:45:36.035523Z","iopub.execute_input":"2023-08-06T17:45:36.035871Z","iopub.status.idle":"2023-08-06T17:45:36.044597Z","shell.execute_reply.started":"2023-08-06T17:45:36.035846Z","shell.execute_reply":"2023-08-06T17:45:36.043736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Landmarksof:\n    def __init__(self, df, frame):\n        f = df[df.frame == frame]\n        face = landmark_pb2.NormalizedLandmarkList()\n        for i in range(0, 468):\n            face.landmark.add(x = f[f\"x_face_{i}\"].iloc[0], y = f[f\"y_face_{i}\"].iloc[0], z = f[f\"z_face_{i}\"].iloc[0])\n        left_hand = landmark_pb2.NormalizedLandmarkList()\n        for i in range(0, 21):\n            left_hand.landmark.add(x = f[f\"x_left_hand_{i}\"].iloc[0], y = f[f\"y_left_hand_{i}\"].iloc[0], z = f[f\"z_left_hand_{i}\"].iloc[0])\n        pose = landmark_pb2.NormalizedLandmarkList()\n        for i in range(0, 33):\n            pose.landmark.add(x = f[f\"x_pose_{i}\"].iloc[0], y = f[f\"y_pose_{i}\"].iloc[0], z = f[f\"z_pose_{i}\"].iloc[0])\n        right_hand = landmark_pb2.NormalizedLandmarkList()\n        for i in range(0, 21):\n            right_hand.landmark.add(x = f[f\"x_right_hand_{i}\"].iloc[0], y = f[f\"y_right_hand_{i}\"].iloc[0], z = f[f\"z_right_hand_{i}\"].iloc[0])\n        self.face_landmarks = face\n        self.left_hand_landmarks = left_hand\n        self.pose_landmarks = pose\n        self.right_hand_landmarks = right_hand","metadata":{"execution":{"iopub.status.busy":"2023-08-06T17:50:08.928906Z","iopub.execute_input":"2023-08-06T17:50:08.929296Z","iopub.status.idle":"2023-08-06T17:50:08.938533Z","shell.execute_reply.started":"2023-08-06T17:50:08.929269Z","shell.execute_reply":"2023-08-06T17:50:08.937082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Visual(lme, fname):\n    annotated_image = np.zeros((1024,1024,3),dtype=np.uint8)\n    img = []\n    for i in sorted(lme[\"frame\"].unique().tolist()):\n        landmarks = Landmarksof(lme, i)\n        img.append(draw_landmarks(landmarks,annotated_image))\n    height, width, _ = img[0].shape\n    out = cv2.VideoWriter(f'{fname}.avi',cv2.VideoWriter_fourcc(*'DIVX'), 15, (width, height))\n    [out.write(img[k]) for k in range(0, len(img))]\n    out.release()","metadata":{"execution":{"iopub.status.busy":"2023-08-06T17:44:00.713874Z","iopub.execute_input":"2023-08-06T17:44:00.714312Z","iopub.status.idle":"2023-08-06T17:44:00.722961Z","shell.execute_reply.started":"2023-08-06T17:44:00.714281Z","shell.execute_reply":"2023-08-06T17:44:00.72188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Visual(lme, \"output1.1\")","metadata":{"execution":{"iopub.status.busy":"2023-08-06T17:50:11.332441Z","iopub.execute_input":"2023-08-06T17:50:11.332812Z","iopub.status.idle":"2023-08-06T17:51:33.999747Z","shell.execute_reply.started":"2023-08-06T17:50:11.332786Z","shell.execute_reply":"2023-08-06T17:51:33.998188Z"},"trusted":true},"execution_count":null,"outputs":[]}]}