{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Sign Language Recognition Challenge\n\nThe goal of this competition is to classify American Sign Language (ASL) signs.\n\nThe landmarks were extracted from raw videos with the MediaPipe holistic model and are asked to predict the sign from this data.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nplt.style.use(\"seaborn-colorblind\")","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:31:09.657649Z","iopub.execute_input":"2023-06-25T09:31:09.659418Z","iopub.status.idle":"2023-06-25T09:31:11.436445Z","shell.execute_reply.started":"2023-06-25T09:31:09.659353Z","shell.execute_reply":"2023-06-25T09:31:11.434925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/asl-fingerspelling/ -GFlash --color","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:31:16.415527Z","iopub.execute_input":"2023-06-25T09:31:16.416119Z","iopub.status.idle":"2023-06-25T09:31:17.543117Z","shell.execute_reply.started":"2023-06-25T09:31:16.416073Z","shell.execute_reply":"2023-06-25T09:31:17.541051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data EDA","metadata":{}},{"cell_type":"code","source":"BASE_DIR = '../input/asl-fingerspelling/'\ntrain = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:31:26.382599Z","iopub.execute_input":"2023-06-25T09:31:26.383188Z","iopub.status.idle":"2023-06-25T09:31:26.569465Z","shell.execute_reply.started":"2023-06-25T09:31:26.383142Z","shell.execute_reply":"2023-06-25T09:31:26.568367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd \ntrain = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:31:31.56719Z","iopub.execute_input":"2023-06-25T09:31:31.567727Z","iopub.status.idle":"2023-06-25T09:31:31.756055Z","shell.execute_reply.started":"2023-06-25T09:31:31.567687Z","shell.execute_reply":"2023-06-25T09:31:31.754949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train.csv has the path to each parquet file, the particpant id, sequence_id and sign.\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:31:52.214256Z","iopub.execute_input":"2023-06-25T09:31:52.21467Z","iopub.status.idle":"2023-06-25T09:31:52.227317Z","shell.execute_reply.started":"2023-06-25T09:31:52.21464Z","shell.execute_reply":"2023-06-25T09:31:52.226325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **WHAT PHRASES ARE WE TRYING TO PREDICT**\n- There are 46518 unique phrases\n- Ranging from 1 to 17 examples of each  ","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"phrase\"].value_counts().head(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Top 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:32:09.64714Z","iopub.execute_input":"2023-06-25T09:32:09.647589Z","iopub.status.idle":"2023-06-25T09:32:10.571862Z","shell.execute_reply.started":"2023-06-25T09:32:09.647559Z","shell.execute_reply":"2023-06-25T09:32:10.570854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"phrase\"].value_counts().tail(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Bottom 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:32:15.218371Z","iopub.execute_input":"2023-06-25T09:32:15.218791Z","iopub.status.idle":"2023-06-25T09:32:16.022076Z","shell.execute_reply.started":"2023-06-25T09:32:15.218758Z","shell.execute_reply":"2023-06-25T09:32:16.020745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parquet Landmark Data\n- Each Parquet file is in the path:\n - train_landmark_files/[train/supplemental].parquet\n - The parquet's associated phrase can be found in train.csv","metadata":{}},{"cell_type":"markdown","source":"# Pull an example parquet file data...\n\nWe pull an example landmark file for the phrase \"surprise az\"","metadata":{}},{"cell_type":"code","source":"example_fn = train.query('phrase == \"surprise az\"')[\"path\"].values[0]\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")\nexample_landmark.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:32:23.456163Z","iopub.execute_input":"2023-06-25T09:32:23.456572Z","iopub.status.idle":"2023-06-25T09:32:39.398246Z","shell.execute_reply.started":"2023-06-25T09:32:23.45654Z","shell.execute_reply":"2023-06-25T09:32:39.397121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:32:56.973424Z","iopub.execute_input":"2023-06-25T09:32:56.973846Z","iopub.status.idle":"2023-06-25T09:32:57.037611Z","shell.execute_reply.started":"2023-06-25T09:32:56.973812Z","shell.execute_reply":"2023-06-25T09:32:57.036284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_frames = example_landmark[\"frame\"].nunique()\n\nprint(\n    f\"The file has {unique_frames} unique frames\"\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:33:00.58888Z","iopub.execute_input":"2023-06-25T09:33:00.589367Z","iopub.status.idle":"2023-06-25T09:33:00.601506Z","shell.execute_reply.started":"2023-06-25T09:33:00.589336Z","shell.execute_reply":"2023-06-25T09:33:00.599727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets Compare for a bunch of parquet files what type of data we have.\n- We notice the number of frames is not consistent\n- Almost every file has 4 types of landmarks: face, left_hand, pose and right_hand.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nBASE_DIR = '../input/asl-fingerspelling/'\ntrain = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\nsurprise_az_files = train.query('phrase == \"surprise az\"')[\"path\"].values\nfor i, f in enumerate(surprise_az_files):\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{f}\")\n    unique_frames = example_landmark[\"frame\"].nunique()\n    print(f\"The file has {unique_frames} unique frames \")\n    if i == 20:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:33:06.455248Z","iopub.execute_input":"2023-06-25T09:33:06.455692Z","iopub.status.idle":"2023-06-25T09:36:33.422013Z","shell.execute_reply.started":"2023-06-25T09:33:06.455661Z","shell.execute_reply":"2023-06-25T09:36:33.420463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the training data provided to us, let us see how many instances are there for surprise az phrase","metadata":{}},{"cell_type":"code","source":"train.query('phrase == \"surprise az\"')","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:37:48.902397Z","iopub.execute_input":"2023-06-25T09:37:48.902863Z","iopub.status.idle":"2023-06-25T09:37:48.933275Z","shell.execute_reply.started":"2023-06-25T09:37:48.90283Z","shell.execute_reply":"2023-06-25T09:37:48.932089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.query('phrase == \"surprise az\"')[\"sequence_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:38:26.804514Z","iopub.execute_input":"2023-06-25T09:38:26.804998Z","iopub.status.idle":"2023-06-25T09:38:26.822396Z","shell.execute_reply.started":"2023-06-25T09:38:26.804967Z","shell.execute_reply":"2023-06-25T09:38:26.821288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 17 unique sequence IDs for surprise phrase\nChoose any sequence ID and load its path(surprise), whose parquet file we will use afterwards","metadata":{}},{"cell_type":"code","source":"surprise = train.query('phrase == \"surprise az\"')[\"path\"].values[0]\nsurprised = pd.read_parquet(f\"{BASE_DIR}{surprise}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:39:00.474774Z","iopub.execute_input":"2023-06-25T09:39:00.475215Z","iopub.status.idle":"2023-06-25T09:39:08.338892Z","shell.execute_reply.started":"2023-06-25T09:39:00.475184Z","shell.execute_reply":"2023-06-25T09:39:08.337752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"surprised","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:39:22.437566Z","iopub.execute_input":"2023-06-25T09:39:22.438067Z","iopub.status.idle":"2023-06-25T09:39:22.501817Z","shell.execute_reply.started":"2023-06-25T09:39:22.438032Z","shell.execute_reply":"2023-06-25T09:39:22.500531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"surprised.index.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:39:58.426312Z","iopub.execute_input":"2023-06-25T09:39:58.426834Z","iopub.status.idle":"2023-06-25T09:39:58.437361Z","shell.execute_reply.started":"2023-06-25T09:39:58.426793Z","shell.execute_reply":"2023-06-25T09:39:58.435745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are total of 999 unique sequence Ids for first parquet file of surprised phrase","metadata":{}},{"cell_type":"code","source":"selected_id = surprised.index.values[10]\nprint(selected_id)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:40:29.752065Z","iopub.execute_input":"2023-06-25T09:40:29.752506Z","iopub.status.idle":"2023-06-25T09:40:29.758924Z","shell.execute_reply.started":"2023-06-25T09:40:29.752478Z","shell.execute_reply":"2023-06-25T09:40:29.757945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"let's see how many instances are there for this sequence id","metadata":{}},{"cell_type":"code","source":"selected_id_df = surprised[surprised.index == selected_id]\nselected_id_df","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:41:12.753148Z","iopub.execute_input":"2023-06-25T09:41:12.753657Z","iopub.status.idle":"2023-06-25T09:41:12.800037Z","shell.execute_reply.started":"2023-06-25T09:41:12.753625Z","shell.execute_reply":"2023-06-25T09:41:12.798197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are total of 186 instances for the selected sequence Id","metadata":{}},{"cell_type":"markdown","source":"Create a function which will seperate all the columns for loaded parquet file along three axes - x, y, z","metadata":{}},{"cell_type":"code","source":"def x_y_z(columns):\n    x = [col for col in columns if col.startswith(\"x\")]\n    y = [col for col in columns if col.startswith(\"y\")]\n    z = [col for col in columns if col.startswith(\"z\")]\n    return x, y, z","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:42:28.274053Z","iopub.execute_input":"2023-06-25T09:42:28.274517Z","iopub.status.idle":"2023-06-25T09:42:28.281775Z","shell.execute_reply.started":"2023-06-25T09:42:28.274484Z","shell.execute_reply":"2023-06-25T09:42:28.280402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating a function which will store types of landmarks present for that id","metadata":{}},{"cell_type":"code","source":"def type_of_landmark(example_landmark):\n    body_parts = set()\n    for col in example_landmark.columns:\n        parts = col.split(\"_\")\n        if len(parts) >= 2:\n            if parts[1] == \"right\":\n                body_parts.add(\"right_hand\")\n            elif parts[1] == \"left\":\n                body_parts.add(\"left_hand\")\n            else:\n                body_parts.add(parts[1])\n    return body_parts","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:43:06.083054Z","iopub.execute_input":"2023-06-25T09:43:06.083578Z","iopub.status.idle":"2023-06-25T09:43:06.09176Z","shell.execute_reply.started":"2023-06-25T09:43:06.083541Z","shell.execute_reply":"2023-06-25T09:43:06.090442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from colorama import Style, Fore\n\nblk = Style.BRIGHT + Fore.BLACK\nred = Style.BRIGHT + Fore.RED\nblu = Style.BRIGHT + Fore.BLUE\ncyan = Style.BRIGHT + Fore.CYAN\ngreen = Style.BRIGHT + Fore.GREEN\nres = Style.RESET_ALL","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:45:29.097809Z","iopub.execute_input":"2023-06-25T09:45:29.098394Z","iopub.status.idle":"2023-06-25T09:45:29.110743Z","shell.execute_reply.started":"2023-06-25T09:45:29.098358Z","shell.execute_reply":"2023-06-25T09:45:29.109231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_frames = selected_id_df[\"frame\"].nunique()\n# type of landmarks/body parts = [face/pose/right hand/left hand]\ntype_landmark_train = type_of_landmark(selected_id_df)\n\n# seperate columns according to landmarks\nface_train = [col for col in selected_id_df.columns if \"face\" in col]\nright_hand_train = [col for col in selected_id_df.columns if \"right_hand\" in col]\nleft_hand_train = [col for col in selected_id_df.columns if \"left_hand\" in col]\npose_train = [col for col in selected_id_df.columns if \"pose\" in col]\n\n# use the function created earlier to distribute along axes\nx_face_train, y_face_train, z_face_train = x_y_z(face_train)\nx_right_hand, y_right_hand, z_right_hand = x_y_z(right_hand_train)\nx_left_hand, y_left_hand, z_left_hand = x_y_z(left_hand_train)\nx_pose, y_pose, z_pose = x_y_z(pose_train)\n\n# Summary\nprint(f'{cyan}{\"*\"*30} Training Data {\"*\"*30}')\nprint(f\"{blk}Selected Sequence ID: {red}{selected_id}\")\nprint(f\"{blk}Unique Frames: {red} {unique_frames}\")\nprint(\n    f\"{blk}Selected example has total {red}{len(type_landmark_train)}{blk} landmards and they are{red}{type_landmark_train}\"\n)\nprint(f\"{blk}{red}\")\nprint(f'{green}{\"*\"*20}FACE{\"*\"*20}')\nprint(f\"{blk}Total Keypoints: {red}{len(face_train)}\")\nprint(f\"{blk}Keypoints in X: {red}{len(x_face_train)}\")\nprint(f\"{blk}Keypoints in Y: {red}{len(y_face_train)}\")\nprint(f\"{blk}Keypoints in Z: {red}{len(z_face_train)}\")\n\nprint(f'{green}{\"*\"*20}RIGHT HAND{\"*\"*20}')\nprint(f\"{blk}Total Keypoints: {red}{len(right_hand_train)}\")\nprint(f\"{blk}Keypoints in X: {red}{len(x_right_hand)}\")\nprint(f\"{blk}Keypoints in Y: {red}{len(y_right_hand)}\")\nprint(f\"{blk}Keypoints in Z: {red}{len(z_right_hand)}\")\n\nprint(f'{green}{\"*\"*20}LEFT HAND{\"*\"*20}')\nprint(f\"{blk}Total Keypoints: {red}{len(left_hand_train)}\")\nprint(f\"{blk}Keypoints in X: {red}{len(x_left_hand)}\")\nprint(f\"{blk}Keypoints in Y: {red}{len(y_left_hand)}\")\nprint(f\"{blk}Keypoints in Z: {red}{len(z_left_hand)}\")\n\nprint(f'{green}{\"*\"*20}POSE{\"*\"*20}')\nprint(f\"{blk}Total Keypoints: {red}{len(pose_train)}\")\nprint(f\"{blk}Keypoints in X: {red}{len(x_pose)}\")\nprint(f\"{blk}Keypoints in Y: {red}{len(y_pose)}\")\nprint(f\"{blk}Keypoints in Z: {red}{len(z_pose)}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:45:33.577506Z","iopub.execute_input":"2023-06-25T09:45:33.578385Z","iopub.status.idle":"2023-06-25T09:45:33.604481Z","shell.execute_reply.started":"2023-06-25T09:45:33.578335Z","shell.execute_reply":"2023-06-25T09:45:33.603047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CREATE METADATA FOR THE TRAINING DATASET","metadata":{}},{"cell_type":"code","source":" xyz_meta = example_landmark.agg(\n        {\n            \"x_face_0\":[\"min\", \"max\", \"mean\"],\n            \"x_face_1\":[\"min\", \"max\", \"mean\"], \n            \"x_face_2\":[\"min\", \"max\", \"mean\"], \n            \"x_face_3\":[\"min\", \"max\", \"mean\"], \n            \"y_left_hand_0\":[\"min\", \"max\", \"mean\"], \n            \"y_left_hand_1\":[\"min\", \"max\", \"mean\"], \n            \"y_left_hand_2\":[\"min\", \"max\", \"mean\"], \n            \"y_left_hand_3\":[\"min\", \"max\", \"mean\"], \n            \"z_right_hand_0\":[\"min\", \"max\", \"mean\"],\n            \"z_right_hand_1\":[\"min\", \"max\", \"mean\"], \n            \"z_right_hand_2\":[\"min\", \"max\", \"mean\"], \n            \"z_right_hand_3\":[\"min\", \"max\", \"mean\"],\n        }\n    )","metadata":{"execution":{"iopub.status.busy":"2023-06-18T14:59:18.236739Z","iopub.execute_input":"2023-06-18T14:59:18.237161Z","iopub.status.idle":"2023-06-18T14:59:18.282461Z","shell.execute_reply.started":"2023-06-18T14:59:18.237126Z","shell.execute_reply":"2023-06-18T14:59:18.281342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_PARQUETS_TO_READ = 1_000\nimport pandas as pd\nfrom tqdm import tqdm\n\nBASE_DIR = '../input/asl-fingerspelling/'\ntrain = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n\ncombined_meta = {}\nfor i, d in tqdm(train.iterrows(), total=len(train)):\n    file_path = d[\"path\"]\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{file_path}\")\n    meta = example_landmark.dropna(subset=['x_face_0', 'x_face_1', 'x_face_2', 'x_face_3', 'x_face_4',\n       'x_face_5', 'x_face_6', 'x_face_7', 'x_face_8',\n       'z_right_hand_11', 'z_right_hand_12', 'z_right_hand_13',\n       'z_right_hand_14', 'z_right_hand_15', 'z_right_hand_16',\n       'z_right_hand_17', 'z_right_hand_18', 'z_right_hand_19',\n       'z_right_hand_20'])[\"frame\"].value_counts().to_dict()\n    meta[\"frames\"] = example_landmark[\"frame\"].nunique()\n    \n    xyz_meta = (\n        example_landmark.agg(\n        {\n            \"x_face_0\":[\"min\", \"max\", \"mean\"],\n            \"x_face_1\":[\"min\", \"max\", \"mean\"], \n            \"x_face_2\":[\"min\", \"max\", \"mean\"], \n            \"x_face_3\":[\"min\", \"max\", \"mean\"], \n            \"y_left_hand_0\":[\"min\", \"max\", \"mean\"], \n            \"y_left_hand_1\":[\"min\", \"max\", \"mean\"], \n            \"y_left_hand_2\":[\"min\", \"max\", \"mean\"], \n            \"y_left_hand_3\":[\"min\", \"max\", \"mean\"], \n            \"z_right_hand_0\":[\"min\", \"max\", \"mean\"],\n            \"z_right_hand_1\":[\"min\", \"max\", \"mean\"], \n            \"z_right_hand_2\":[\"min\", \"max\", \"mean\"], \n            \"z_right_hand_3\":[\"min\", \"max\", \"mean\"],\n        }\n    )\n    .unstack().to_dict()\n    )\n    \n    for key in xyz_meta.keys():\n        new_key = key[0] + \"_\" + key[1]\n        meta[new_key] = xyz_meta[key]\n    \n    combined_meta[file_path] = meta\n    \n    if i >= N_PARQUETS_TO_READ:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-06-18T14:59:21.66315Z","iopub.execute_input":"2023-06-18T14:59:21.664321Z","iopub.status.idle":"2023-06-18T15:49:32.272598Z","shell.execute_reply.started":"2023-06-18T14:59:21.664246Z","shell.execute_reply":"2023-06-18T15:49:32.271004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xyz_meta","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:49:59.955662Z","iopub.execute_input":"2023-06-18T15:49:59.956094Z","iopub.status.idle":"2023-06-18T15:49:59.967018Z","shell.execute_reply.started":"2023-06-18T15:49:59.956063Z","shell.execute_reply":"2023-06-18T15:49:59.965963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ncombined_meta = {}\ntrain = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\ntrain_with_meta = train.merge(\n    pd.DataFrame(combined_meta).T.reset_index().rename(columns={\"index\": \"path\"}),\n    how=\"left\",\n)\ntrain_with_meta.to_parquet(\"train_with_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-06-18T17:35:05.155433Z","iopub.execute_input":"2023-06-18T17:35:05.155863Z","iopub.status.idle":"2023-06-18T17:35:05.362538Z","shell.execute_reply.started":"2023-06-18T17:35:05.155826Z","shell.execute_reply":"2023-06-18T17:35:05.361178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"What are the most frequent types of landmarks provided?\n- face has a lot more datapoints because mediapipe provides 468 3D datapoints per frame.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n# Exclude columns with mixed data types from sorting\ndesired_columns = ['path', 'file_id', 'sequence_id', 'participant_id', 'phrase']\nnumerical_columns = [col for col in desired_columns if train_with_meta[col].dtype != object]\n# Plot the graph\ntrain_with_meta[numerical_columns].sum().sort_values().plot(kind=\"barh\", title=\"Sum of Rows by Landmark Type\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:50:12.813489Z","iopub.execute_input":"2023-06-18T15:50:12.813944Z","iopub.status.idle":"2023-06-18T15:50:13.03223Z","shell.execute_reply.started":"2023-06-18T15:50:12.813907Z","shell.execute_reply":"2023-06-18T15:50:13.031135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n# Define the custom comparison function\ndef custom_comparison(x):\n    if pd.api.types.is_numeric_dtype(x):\n        return x > 0\n    else:\n        return x\n# Perform the comparison\ncomparison_result = train_with_meta.query('index < 1000').fillna(0)[\n    [\"path\", \"file_id\", \"sequence_id\", \"participant_id\", \"phrase\"]\n].apply(custom_comparison)\nmean_result = comparison_result.mean().plot(kind = 'barh', title = 'Percent of Frame/keypoints with Data')\nprint(mean_result)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:50:16.794344Z","iopub.execute_input":"2023-06-18T15:50:16.794733Z","iopub.status.idle":"2023-06-18T15:50:17.028653Z","shell.execute_reply.started":"2023-06-18T15:50:16.794704Z","shell.execute_reply":"2023-06-18T15:50:17.027483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.dropna(subset=['x_face_0', 'x_face_1', 'x_face_2', 'x_face_3', 'x_face_4',\n       'x_face_5', 'x_face_6', 'x_face_7', 'x_face_8',\n       \n       'z_right_hand_11', 'z_right_hand_12', 'z_right_hand_13',\n       'z_right_hand_14', 'z_right_hand_15', 'z_right_hand_16',\n       'z_right_hand_17', 'z_right_hand_18', 'z_right_hand_19',\n       'z_right_hand_20'])[\"frame\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:50:21.105296Z","iopub.execute_input":"2023-06-18T15:50:21.105724Z","iopub.status.idle":"2023-06-18T15:50:21.582967Z","shell.execute_reply.started":"2023-06-18T15:50:21.105689Z","shell.execute_reply":"2023-06-18T15:50:21.581867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(example_landmark.columns)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:50:25.000387Z","iopub.execute_input":"2023-06-18T15:50:25.0008Z","iopub.status.idle":"2023-06-18T15:50:25.008529Z","shell.execute_reply.started":"2023-06-18T15:50:25.000768Z","shell.execute_reply":"2023-06-18T15:50:25.007252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pattern = '^(x_|y_|z_)'\nfiltered_columns = example_landmark.filter(regex=pattern)\nexample_landmark.dropna(subset=filtered_columns.columns)[\"frame\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:50:28.333495Z","iopub.execute_input":"2023-06-18T15:50:28.333906Z","iopub.status.idle":"2023-06-18T15:50:30.467688Z","shell.execute_reply.started":"2023-06-18T15:50:28.333875Z","shell.execute_reply":"2023-06-18T15:50:30.466509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = [\"file_id\", \"sequence_id\", \"participant_id\", \"phrase\"]\ntrain_with_meta.dropna(subset=[\"path\"])[columns].apply(pd.to_numeric, errors='coerce') > 0","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:50:33.493815Z","iopub.execute_input":"2023-06-18T15:50:33.494363Z","iopub.status.idle":"2023-06-18T15:50:33.651026Z","shell.execute_reply.started":"2023-06-18T15:50:33.494315Z","shell.execute_reply":"2023-06-18T15:50:33.649826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Every parquet file has at least some datapoints for all four types of landmarks:**\n\n- Face, pose, left hand and right hand.","metadata":{}},{"cell_type":"markdown","source":"# Check one example ","metadata":{}},{"cell_type":"code","source":"import pandas as pd \nBASE_DIR = '../input/asl-fingerspelling/'\ntrain = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\nexample_fn = train.query('phrase == \"surprise az\"')[\"path\"].values[0]\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:50:38.670782Z","iopub.execute_input":"2023-06-18T15:50:38.671236Z","iopub.status.idle":"2023-06-18T15:50:48.395579Z","shell.execute_reply.started":"2023-06-18T15:50:38.671201Z","shell.execute_reply":"2023-06-18T15:50:48.394576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"no_xyz\"] = example_landmark['x_face_0'].isna()","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:50:52.184794Z","iopub.execute_input":"2023-06-18T15:50:52.185186Z","iopub.status.idle":"2023-06-18T15:50:52.191618Z","shell.execute_reply.started":"2023-06-18T15:50:52.185156Z","shell.execute_reply":"2023-06-18T15:50:52.190691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.groupby(\"frame\")[\"no_xyz\"].sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:50:54.981885Z","iopub.execute_input":"2023-06-18T15:50:54.982311Z","iopub.status.idle":"2023-06-18T15:50:55.005021Z","shell.execute_reply.started":"2023-06-18T15:50:54.982277Z","shell.execute_reply":"2023-06-18T15:50:55.004104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.groupby(\"frame\")[\"no_xyz\"].sum().plot(\n    title=\"missing xyz per frame\", kind=\"bar\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:51:06.160065Z","iopub.execute_input":"2023-06-18T15:51:06.160477Z","iopub.status.idle":"2023-06-18T15:51:11.543943Z","shell.execute_reply.started":"2023-06-18T15:51:06.160447Z","shell.execute_reply":"2023-06-18T15:51:11.542789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"frame\"].median()","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:51:15.793889Z","iopub.execute_input":"2023-06-18T15:51:15.794362Z","iopub.status.idle":"2023-06-18T15:51:15.806546Z","shell.execute_reply.started":"2023-06-18T15:51:15.794325Z","shell.execute_reply":"2023-06-18T15:51:15.805192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3D plot of Landmarks from \"surprise_az\" example\n\nPick frame 578 because we have no missing xyz data","metadata":{}},{"cell_type":"code","source":"example_frame = example_landmark.query(\"frame == 578\")","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:51:19.371578Z","iopub.execute_input":"2023-06-18T15:51:19.372005Z","iopub.status.idle":"2023-06-18T15:51:19.472139Z","shell.execute_reply.started":"2023-06-18T15:51:19.371972Z","shell.execute_reply":"2023-06-18T15:51:19.471214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\nimport pandas as pd\ndf = pd.DataFrame({\n    'frame': [578, 578, 578, 578, 578],\n    'x_face': [0.574436, 0.620399, 0.550710, 0.615361, 0.473677],\n    'y_face': [0.560474, 0.604428, 0.539382, 0.603240, 0.468712],\n    'z_face': [0.566295, 0.609184, 0.545007, 0.609855, 0.470851],\n    'x_right_hand': [0.482442, 0.475004, 0.463145, 0.520459, 0.327291],\n    'y_right_hand': [0.563047, 0.598960, 0.537619, 0.608835, 0.478398],\n    'z_right_hand': [0.543066, -0.058136, 0.550710, 0.613422, -0.171806]\n})\n\nfig = px.scatter_3d(df, x='x_face', y='y_face', z='z_face')\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T15:51:27.227952Z","iopub.execute_input":"2023-06-18T15:51:27.228427Z","iopub.status.idle":"2023-06-18T15:51:30.011909Z","shell.execute_reply.started":"2023-06-18T15:51:27.228392Z","shell.execute_reply":"2023-06-18T15:51:30.010728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Try to draw the Mediapipe's hand connections?","metadata":{}},{"cell_type":"code","source":"!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:46:56.368428Z","iopub.execute_input":"2023-06-25T09:46:56.369488Z","iopub.status.idle":"2023-06-25T09:47:15.547327Z","shell.execute_reply.started":"2023-06-25T09:46:56.369439Z","shell.execute_reply":"2023-06-25T09:47:15.546035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\nmp_hands = mp.solutions.hands\nmp_hands.HAND_CONNECTIONS","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:47:40.04658Z","iopub.execute_input":"2023-06-25T09:47:40.047153Z","iopub.status.idle":"2023-06-25T09:47:51.38154Z","shell.execute_reply.started":"2023-06-25T09:47:40.047114Z","shell.execute_reply":"2023-06-25T09:47:51.379838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_id_df.query(\"sequence_id == @selected_id and frame == 12\")","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:48:17.830269Z","iopub.execute_input":"2023-06-25T09:48:17.831319Z","iopub.status.idle":"2023-06-25T09:48:17.955271Z","shell.execute_reply.started":"2023-06-25T09:48:17.831278Z","shell.execute_reply":"2023-06-25T09:48:17.953941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_id_df.query(\"sequence_id == @selected_id and frame == 2\")[x_face_train]","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:48:34.783765Z","iopub.execute_input":"2023-06-25T09:48:34.784214Z","iopub.status.idle":"2023-06-25T09:48:34.856047Z","shell.execute_reply.started":"2023-06-25T09:48:34.784181Z","shell.execute_reply":"2023-06-25T09:48:34.854794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_id_df.query(\"sequence_id == @selected_id and frame == 2\")[x_face_train].iloc[\n    0\n].values","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:48:49.11349Z","iopub.execute_input":"2023-06-25T09:48:49.113973Z","iopub.status.idle":"2023-06-25T09:48:49.177852Z","shell.execute_reply.started":"2023-06-25T09:48:49.113939Z","shell.execute_reply":"2023-06-25T09:48:49.176626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(\n    selected_id_df.query(\"sequence_id == @selected_id and frame == 2\")[x_face_train]\n    .iloc[0]\n    .values\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:49:07.226812Z","iopub.execute_input":"2023-06-25T09:49:07.228201Z","iopub.status.idle":"2023-06-25T09:49:07.287921Z","shell.execute_reply.started":"2023-06-25T09:49:07.228141Z","shell.execute_reply":"2023-06-25T09:49:07.286142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(\n    [\n        int(col.split(\"_\")[-1])\n        for col in selected_id_df.query(\"sequence_id == @selected_id and frame == 12\")[\n            face_train\n        ].columns\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:49:28.126668Z","iopub.execute_input":"2023-06-25T09:49:28.12783Z","iopub.status.idle":"2023-06-25T09:49:28.189444Z","shell.execute_reply.started":"2023-06-25T09:49:28.127767Z","shell.execute_reply":"2023-06-25T09:49:28.188017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_plot(seq, frame, x_col, y_col, z_col, df):\n    x = df.query(\"sequence_id == @seq and frame == @frame\")[x_col].iloc[0].values\n    y = df.query(\"sequence_id == @seq and frame == @frame\")[y_col].iloc[0].values\n    z = df.query(\"sequence_id == @seq and frame == @frame\")[z_col].iloc[0].values\n\n    landmark_idx = [\n        int(col.split(\"_\")[-1])\n        for col in df.query(\"sequence_id == @seq and frame == @frame\")[x_col].columns\n    ]\n\n    dataframe = pd.DataFrame({\"x\": x, \"y\": y, \"z\": z, \"landmark_idx\": landmark_idx})\n\n    return dataframe","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:49:43.448313Z","iopub.execute_input":"2023-06-25T09:49:43.448734Z","iopub.status.idle":"2023-06-25T09:49:43.457514Z","shell.execute_reply.started":"2023-06-25T09:49:43.448704Z","shell.execute_reply":"2023-06-25T09:49:43.456008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame = 3\nleft_hand_train = data_plot(\n    selected_id, frame, x_left_hand, y_left_hand, z_left_hand, selected_id_df\n)\nright_hand_train = data_plot(\n    selected_id, frame, x_right_hand, y_right_hand, z_right_hand, selected_id_df\n)\nface_train = data_plot(\n    selected_id, frame, x_face_train, y_face_train, z_face_train, selected_id_df\n)\npose_train = data_plot(selected_id, frame, x_pose, y_pose, z_pose, selected_id_df)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:50:07.726568Z","iopub.execute_input":"2023-06-25T09:50:07.727049Z","iopub.status.idle":"2023-06-25T09:50:08.448937Z","shell.execute_reply.started":"2023-06-25T09:50:07.727018Z","shell.execute_reply":"2023-06-25T09:50:08.447581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(5, 5))\nax.scatter(right_hand_train[\"x\"], right_hand_train[\"y\"])\nax.scatter(left_hand_train[\"x\"], left_hand_train[\"y\"])\nax.scatter(face_train[\"x\"], face_train[\"y\"])\nax.scatter(pose_train[\"x\"], pose_train[\"y\"])\n\n\nfor connection in mp_hands.HAND_CONNECTIONS:\n    point_a = connection[0]\n    point_b = connection[1]\n    x1, y1 = right_hand_train.query(\"landmark_idx == @point_a\")[[\"x\", \"y\"]].values[0]\n    x2, y2 = right_hand_train.query(\"landmark_idx == @point_b\")[[\"x\", \"y\"]].values[0]\n    plt.plot([x1, x2], [y1, y2], color=\"red\")\n    x3, y3 = left_hand_train.query(\"landmark_idx == @point_a\")[[\"x\", \"y\"]].values[0]\n    x4, y4 = left_hand_train.query(\"landmark_idx == @point_b\")[[\"x\", \"y\"]].values[0]\n    plt.plot([x3, x4], [y3, y4], color=\"red\")\n    x5, y5 = face_train.query(\"landmark_idx == @point_a\")[[\"x\", \"y\"]].values[0]\n    x6, y6 = face_train.query(\"landmark_idx == @point_b\")[[\"x\", \"y\"]].values[0]\n    plt.plot([x5, x6], [y5, y6], color=\"blue\")\n    x7, y7 = pose_train.query(\"landmark_idx == @point_a\")[[\"x\", \"y\"]].values[0]\n    x8, y8 = pose_train.query(\"landmark_idx == @point_b\")[[\"x\", \"y\"]].values[0]\n    plt.plot([x7, x8], [y7, y8], color=\"blue\")\n\nax.set_title(\"Surprised\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:50:24.536614Z","iopub.execute_input":"2023-06-25T09:50:24.537148Z","iopub.status.idle":"2023-06-25T09:50:25.555631Z","shell.execute_reply.started":"2023-06-25T09:50:24.537111Z","shell.execute_reply":"2023-06-25T09:50:25.554019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(5, 5))\nax.scatter(right_hand_train[\"x\"], right_hand_train[\"y\"])\nax.scatter(left_hand_train[\"x\"], left_hand_train[\"y\"])\n# ax.scatter(face_train[\"x\"], face_train[\"y\"])\n# ax.scatter(pose_train[\"x\"], pose_train[\"y\"])\n\n\nfor connection in mp_hands.HAND_CONNECTIONS:\n    point_a = connection[0]\n    point_b = connection[1]\n    x1, y1 = right_hand_train.query(\"landmark_idx == @point_a\")[[\"x\", \"y\"]].values[0]\n    x2, y2 = right_hand_train.query(\"landmark_idx == @point_b\")[[\"x\", \"y\"]].values[0]\n    plt.plot([x1, x2], [y1, y2], color=\"red\")\n    x3, y3 = left_hand_train.query(\"landmark_idx == @point_a\")[[\"x\", \"y\"]].values[0]\n    x4, y4 = left_hand_train.query(\"landmark_idx == @point_b\")[[\"x\", \"y\"]].values[0]\n    plt.plot([x3, x4], [y3, y4], color=\"red\")\n\nax.set_title(\"Surprised\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:51:00.402569Z","iopub.execute_input":"2023-06-25T09:51:00.403083Z","iopub.status.idle":"2023-06-25T09:51:01.076679Z","shell.execute_reply.started":"2023-06-25T09:51:00.403049Z","shell.execute_reply":"2023-06-25T09:51:01.075302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(5, 5))\n# ax.scatter(right_hand_train[\"x\"], right_hand_train[\"y\"])\n# ax.scatter(left_hand_train[\"x\"], left_hand_train[\"y\"])\nax.scatter(face_train[\"x\"], face_train[\"y\"])\nax.scatter(pose_train[\"x\"], pose_train[\"y\"])\n\n\nfor connection in mp_hands.HAND_CONNECTIONS:\n    point_a = connection[0]\n    point_b = connection[1]\n    x5, y5 = face_train.query(\"landmark_idx == @point_a\")[[\"x\", \"y\"]].values[0]\n    x6, y6 = face_train.query(\"landmark_idx == @point_b\")[[\"x\", \"y\"]].values[0]\n    plt.plot([x5, x6], [y5, y6], color=\"blue\")\n    x7, y7 = pose_train.query(\"landmark_idx == @point_a\")[[\"x\", \"y\"]].values[0]\n    x8, y8 = pose_train.query(\"landmark_idx == @point_b\")[[\"x\", \"y\"]].values[0]\n    plt.plot([x7, x8], [y7, y8], color=\"blue\")\n\nax.set_title(\"Surprised\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:51:43.566285Z","iopub.execute_input":"2023-06-25T09:51:43.566807Z","iopub.status.idle":"2023-06-25T09:51:44.23563Z","shell.execute_reply.started":"2023-06-25T09:51:43.566773Z","shell.execute_reply":"2023-06-25T09:51:44.234156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_new_to_old_style(sequence):\n    # there are 4 types of landmarks [face,pose,right_hand,left_hand]\n    types = []# list where we'll store landmarks\n    landmark_indexes = []\n    for column in list(sequence.columns)[1:544]:\n        # first column is frame which is not a landmark so counter starts from 1\n        # why 544 - because we are given that there are now 1,629 spatial coordinate columns for the x, y and z coordinates for each of the 543 landmarks.\n        parts = column.split(\"_\")\n        if len(parts) == 4:\n            # for x_left_hand_1 - there will be 4 parts and for x_pose_1 there will be 3 only\n            types.append(parts[1] + \"_\" + parts[2])\n        else:\n            types.append(parts[1])\n\n        landmark_indexes.append(int(parts[-1]))\n\n    data = {\"frame\": [], \"type\": [], \"landmark_index\": [], \"x\": [], \"y\": [], \"z\": []}\n\n    for index, row in sequence.iterrows():\n        data[\"frame\"] += [int(row.frame)] * 543\n        data[\"type\"] += types\n        data[\"landmark_index\"] += landmark_indexes\n\n        for _type, landmark_index in zip(types, landmark_indexes):\n            data[\"x\"].append(row[f\"x_{_type}_{landmark_index}\"])\n            data[\"y\"].append(row[f\"y_{_type}_{landmark_index}\"])\n            data[\"z\"].append(row[f\"z_{_type}_{landmark_index}\"])\n\n    return pd.DataFrame.from_dict(data)\n\n\ndef assign_colors(row):\n    if row == \"face\":\n        return \"red\"\n    elif \"hand\" in row:\n        return \"blue\"\n    else:\n        return \"green\"\n\n\n# specifies the plotting order\ndef assign_order(row):\n    if row.type == \"face\":\n        return row.landmark_index + 101\n    elif row.type == \"pose\":\n        return row.landmark_index + 30\n    elif row.type == \"left_hand\":\n        return row.landmark_index + 80\n    else:\n        return row.landmark_index\n\n\n# A function to visualize the landmarks in 2d\ndef visualise2d_landmarks(parquet_df, title=\"\"):\n    # we first define a list of landmark connections, which specify which landmarks are connected by a line.\n\n    # face landmarks are not connected by lines, you can also see that we have added 101 to face landmark indexs and in connections all values are below 100\n\n    connections = [  \n        [0, 1, 2, 3, 4,],\n        [0, 5, 6, 7, 8],\n        [0, 9, 10, 11, 12],\n        [0, 13, 14, 15, 16],\n        [0, 17, 18, 19, 20],\n\n        \n        [38, 36, 35, 34, 30, 31, 32, 33, 37],\n        [40, 39],\n        [52, 46, 50, 48, 46, 44, 42, 41, 43, 45, 47, 49, 45, 51],\n        [42, 54, 56, 58, 60, 62, 58],\n        [41, 53, 55, 57, 59, 61, 57],\n        [54, 53],\n\n        \n        [80, 81, 82, 83, 84, ],\n        [80, 85, 86, 87, 88],\n        [80, 89, 90, 91, 92],\n        [80, 93, 94, 95, 96],\n        [80, 97, 98, 99, 100], ]\n\n    parquet_df = map_new_to_old_style(parquet_df)\n    frames = sorted(set(parquet_df.frame))\n    first_frame = min(frames)\n    parquet_df[\"color\"] = parquet_df.type.apply(lambda row: assign_colors(row))\n    parquet_df[\"plot_order\"] = parquet_df.apply(lambda row: assign_order(row), axis=1)\n    first_frame_df = parquet_df[parquet_df.frame == first_frame].copy()\n    first_frame_df = first_frame_df.sort_values([\"plot_order\"]).set_index(\"plot_order\")\n\n    frames_l = []\n    for frame in frames:\n        filtered_df = parquet_df[parquet_df.frame == frame].copy()\n        filtered_df = filtered_df.sort_values([\"plot_order\"]).set_index(\"plot_order\")\n        traces = [\n            go.Scatter(\n                x=filtered_df[\"x\"],\n                y=filtered_df[\"y\"],\n                mode=\"markers\",\n                marker=dict(color=filtered_df.color, size=9),\n            )\n        ]\n\n        for i, seg in enumerate(connections):\n            trace = go.Scatter(\n                x=filtered_df.loc[seg][\"x\"],\n                y=filtered_df.loc[seg][\"y\"],\n                mode=\"lines\",\n            )\n            traces.append(trace)\n        frame_data = go.Frame(data=traces, traces=[i for i in range(17)])\n        frames_l.append(frame_data)\n\n    traces = [\n        go.Scatter(\n            x=first_frame_df[\"x\"],\n            y=first_frame_df[\"y\"],\n            mode=\"markers\",\n            marker=dict(color=first_frame_df.color, size=9),\n        )\n    ]\n    for i, seg in enumerate(connections):\n        trace = go.Scatter(\n            x=first_frame_df.loc[seg][\"x\"],\n            y=first_frame_df.loc[seg][\"y\"],\n            mode=\"lines\",\n            line=dict(color=\"black\", width=2),\n        )\n        traces.append(trace)\n    fig = go.Figure(data=traces, frames=frames_l)\n\n    fig.update_layout(\n        width=500,\n        height=800,\n        scene={\n            \"aspectmode\": \"data\",\n        },\n        updatemenus=[\n            {\n                \"buttons\": [\n                    {\n                        \"args\": [\n                            None,\n                            {\n                                \"frame\": {\"duration\": 100, \"redraw\": True},\n                                \"fromcurrent\": True,\n                                \"transition\": {\"duration\": 0},\n                            },\n                        ],\n                        \"label\": \"&#9654;\",\n                        \"method\": \"animate\",\n                    },\n                    {\n                        \"args\": [\n                            [None],\n                            {\n                                \"frame\": {\"duration\": 0, \"redraw\": False},\n                                \"mode\": \"immediate\",\n                                \"transition\": {\"duration\": 0},\n                            },\n                        ],\n                        \"label\": \"&#9612;&#9612;\",\n                        \"method\": \"animate\",\n                    },\n                ],\n                \"direction\": \"left\",\n                \"pad\": {\"r\": 100, \"t\": 100},\n                \"font\": {\"size\": 20},\n                \"type\": \"buttons\",\n                \"x\": 0.1,\n                \"y\": 0,\n                 }\n        ],\n    )\n    camera = dict(up=dict(x=0, y=-1, z=0), eye=dict(x=0, y=0, z=2.5))\n    fig.update_layout(title_text=title, title_x=0.5)\n    fig.update_layout(scene_camera=camera, showlegend=False)\n    fig.update_layout(\n        xaxis=dict(visible=False),\n        yaxis=dict(visible=False),\n    )\n    fig.update_yaxes(autorange=\"reversed\")\n\n    fig.show()\n\n\ndef get_phrase(df, file_id, sequence_id):\n    return df[\n        np.logical_and(df.file_id == file_id, df.sequence_id == sequence_id)\n    ].phrase.iloc[0]","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:55:12.320834Z","iopub.execute_input":"2023-06-25T09:55:12.3235Z","iopub.status.idle":"2023-06-25T09:55:12.386255Z","shell.execute_reply.started":"2023-06-25T09:55:12.323381Z","shell.execute_reply":"2023-06-25T09:55:12.384098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_id = 474255203\nsequence_id = 53122870\n\nsurprise = f\"/kaggle/input/asl-fingerspelling/train_landmarks/{file_id}.parquet\"\nsurprised = pd.read_parquet(surprise)\n\nsequence = surprised[surprised.index == sequence_id]\nsequence","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:55:34.467831Z","iopub.execute_input":"2023-06-25T09:55:34.469437Z","iopub.status.idle":"2023-06-25T09:55:52.301842Z","shell.execute_reply.started":"2023-06-25T09:55:34.469378Z","shell.execute_reply":"2023-06-25T09:55:52.300273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\nimport pandas as pd\nget_phrase(train, file_id, sequence_id)\nvisualise2d_landmarks(sequence, \"Surprised\")","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:58:19.5326Z","iopub.execute_input":"2023-06-25T09:58:19.533204Z","iopub.status.idle":"2023-06-25T09:58:37.640361Z","shell.execute_reply.started":"2023-06-25T09:58:19.533168Z","shell.execute_reply":"2023-06-25T09:58:37.638991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nframe_number = 578\nlandmark_type = 'face'  # or 'right_hand' or any other valid type\n\n# Filter the DataFrame based on the frame number and landmark type\nfiltered_landmarks = example_landmark[example_landmark['frame'] == frame_number]\n\n# Select the columns for the specific landmark type\nlandmark_columns = [column for column in filtered_landmarks.columns if landmark_type in column]\n\n# Scatter plot the 'x' and 'y' coordinates\nplt.scatter(filtered_landmarks[landmark_columns[::2]], filtered_landmarks[landmark_columns[1::2]])\nplt.xlabel('x')\nplt.ylabel('y')\nplt.title(f'{landmark_type.capitalize()} Landmarks - Frame {frame_number}')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-25T09:52:55.771175Z","iopub.execute_input":"2023-06-25T09:52:55.771635Z","iopub.status.idle":"2023-06-25T09:52:56.49151Z","shell.execute_reply.started":"2023-06-25T09:52:55.771604Z","shell.execute_reply":"2023-06-25T09:52:56.48997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TRY TO USE MEDIAPIPE TO PLOT\n- Pull some example images\n- Run mediapipe holistic to see how it produces the results\n- plot them on the image","metadata":{}},{"cell_type":"code","source":"!wget https://i.ytimg.com/vi/mi9f9zOaqM8/hqdefault.jpg --quiet\n!wget https://purepng.com/public/uploads/large/purepng.com-standing-womenwomenpeoplepersonsfemale-1121525078284xmm0l.png --quiet \n!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:26:45.278037Z","iopub.execute_input":"2023-06-18T16:26:45.278515Z","iopub.status.idle":"2023-06-18T16:27:05.333177Z","shell.execute_reply.started":"2023-06-18T16:26:45.278482Z","shell.execute_reply":"2023-06-18T16:27:05.331682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport mediapipe as mp\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nmp_drawing = mp.solutions.drawing_utils\nmp_drawing_styles = mp.solutions.drawing_styles\nmp_holistic = mp.solutions.holistic\n\n# For static images:\nIMAGE_FILES = [\"hqdefault.jpg\",\n               \"purepng.com-standing-womenwomenpeoplepersonsfemale-1121525078284xmm0l.png\"]\nBG_COLOR = (192, 192, 192)  # gray\n\nwith mp_holistic.Holistic(\n    static_image_mode=True,\n    model_complexity=2,\n    enable_segmentation=True,\n    refine_face_landmarks=True\n) as holistic:\n    for idx, file in enumerate(IMAGE_FILES):\n        image = cv2.imread(file)\n        if image is None:\n            print(f\"Failed to read image file: {file}\")\n            continue\n\n        image_height, image_width, _ = image.shape\n        # Convert the BGR image to RGB before processing.\n        results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n\n        if results.pose_landmarks:\n            print(\n                f'Nose coordinates: ('\n                f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '\n                f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'\n            )\n\n        annotated_image = image.copy()\n        condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1\n        bg_image = np.zeros(image.shape, dtype=np.uint8)\n        bg_image[:] = BG_COLOR\n        annotated_image = np.where(condition, annotated_image, bg_image)\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.face_landmarks,\n            mp_holistic.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style()\n        )\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.pose_landmarks,\n            mp_holistic.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style()\n        )\n        cv2.imwrite(f'/tmp/annotated_image{idx}.png', annotated_image)\n        # Plot pose world landmarks.\n        mp_drawing.plot_landmarks(\n            results.pose_world_landmarks, mp_holistic.POSE_CONNECTIONS\n        )\n\n        # Display annotated image using matplotlib\n        plt.imshow(plt.imread(f\"/tmp/annotated_image{idx}.png\"))\n        plt.show()\n\n# For webcam input:\ncap = cv2.VideoCapture(0)\nwith mp_holistic.Holistic(\n    min_detection_confidence=0.5,\n    min_tracking_confidence=0.5\n) as holistic:\n    while cap.isOpened():\n        success, image = cap.read()\n        if not success:\n            print(\"Ignoring empty camera frame.\")\n            # If loading a video, use 'break' instead of 'continue'.\n            continue\n\n        image.flags.writeable = False\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        results = holistic.process(image)\n\n        # Draw landmark annotation on the image.\n        image.flags.writeable = True\n        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n        mp_drawing.draw_landmarks(\n            image,\n            results.face_landmarks,\n            mp_holistic.FACEMESH_CONTOURS,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_contours_style()\n        )\n        mp_drawing.draw_landmarks(\n            image,\n            results.pose_landmarks,\n            mp_holistic.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style()\n        )\n        # Flip the image horizontally for a selfie-view display.\n        cv2.imshow('MediaPipe Holistic', cv2.flip(image, 1))\n        if cv2.waitKey(5) & 0xFF == 27:\n            cap.release()\n            break\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:27:16.978695Z","iopub.execute_input":"2023-06-18T16:27:16.979148Z","iopub.status.idle":"2023-06-18T16:27:25.759791Z","shell.execute_reply.started":"2023-06-18T16:27:16.97911Z","shell.execute_reply":"2023-06-18T16:27:25.758431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Try to use the same format for plotting of parquet data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nbackground_image = np.zeros([720, 720, 3])\nmp_drawing.draw_landmarks(\n    background_image,\n    results.face_landmarks,\n    mp_holistic.FACEMESH_TESSELATION,\n    landmark_drawing_spec=None,\n    connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n)\nmp_drawing.draw_landmarks(\n    background_image,\n    results.pose_landmarks,\n    mp_holistic.POSE_CONNECTIONS,\n    landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style()\n)\nplt.imshow(background_image)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:27:34.540953Z","iopub.execute_input":"2023-06-18T16:27:34.541428Z","iopub.status.idle":"2023-06-18T16:27:35.084756Z","shell.execute_reply.started":"2023-06-18T16:27:34.54139Z","shell.execute_reply":"2023-06-18T16:27:35.083275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(results.face_landmarks)\nfrom mediapipe.framework.formats import landmark_pb2","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:27:39.595019Z","iopub.execute_input":"2023-06-18T16:27:39.596204Z","iopub.status.idle":"2023-06-18T16:27:39.601442Z","shell.execute_reply.started":"2023-06-18T16:27:39.596163Z","shell.execute_reply":"2023-06-18T16:27:39.59969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\nface_columns = [col for col in example_frame.columns if 'face' in col]\nfiltered_frame = example_frame[\n    (example_frame[face_columns] == 'face').all(axis=1) & ~example_frame[face_columns].isna().any(axis=1)\n]\nprint(filtered_frame)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:27:43.252514Z","iopub.execute_input":"2023-06-18T16:27:43.25297Z","iopub.status.idle":"2023-06-18T16:27:43.273129Z","shell.execute_reply.started":"2023-06-18T16:27:43.252935Z","shell.execute_reply":"2023-06-18T16:27:43.271407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\nmp_drawing = mp.solutions.drawing_utils\nmp_drawing.draw_landmarks(\n    background_image,\n    results.face_landmarks,\n    mp_holistic.FACEMESH_TESSELATION,\n    landmark_drawing_spec=None,\n    connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n)\nplt.imshow(background_image)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:27:47.022973Z","iopub.execute_input":"2023-06-18T16:27:47.023417Z","iopub.status.idle":"2023-06-18T16:27:47.456584Z","shell.execute_reply.started":"2023-06-18T16:27:47.023382Z","shell.execute_reply":"2023-06-18T16:27:47.455358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\nimport pandas as pd\n\n# Assuming your dataset is stored in a pandas DataFrame called 'df'\ndf = pd.DataFrame({\n    'frame': [86, 86, 86, 86, 86],\n    'x_face': [0.574436, 0.620399, 0.550710, 0.615361, 0.473677],\n    'y_face': [0.560474, 0.604428, 0.539382, 0.603240, 0.468712],\n    'z_face': [0.566295, 0.609184, 0.545007, 0.609855, 0.470851],\n    'x_right_hand': [0.482442, 0.475004, 0.463145, 0.520459, 0.327291],\n    'y_right_hand': [0.563047, 0.598960, 0.537619, 0.608835, 0.478398],\n    'z_right_hand': [0.543066, -0.058136, 0.550710, 0.613422, -0.171806],\n    # Add more columns as necessary\n})\n\nfig = go.Figure(data=[go.Scatter3d(\n    x=df['x_face'],\n    y=df['y_face'],\n    z=df['z_face'],\n    mode='markers',\n    marker=dict(\n        size=5,\n        color=df['frame'],  # Color based on the 'frame' column\n        colorscale='Viridis',\n        opacity=0.8\n    )\n)])\n\n# Add more columns to the plot if needed\nfig.add_trace(go.Scatter3d(\n    x=df['x_right_hand'],\n    y=df['y_right_hand'],\n    z=df['z_right_hand'],\n    mode='markers',\n    marker=dict(\n        size=5,\n        color=df['frame'],  # Color based on the 'frame' column\n        colorscale='Viridis',\n        opacity=0.8\n    )\n))\n\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:27:51.540399Z","iopub.execute_input":"2023-06-18T16:27:51.540813Z","iopub.status.idle":"2023-06-18T16:27:51.575823Z","shell.execute_reply.started":"2023-06-18T16:27:51.540781Z","shell.execute_reply":"2023-06-18T16:27:51.574575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TODO\nFigure out how to transform the parquet file data into mediapipe NormalizedLandmarkList","metadata":{}},{"cell_type":"markdown","source":"**EVALUATION**","metadata":{}},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=selected_columns)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:28:00.192488Z","iopub.execute_input":"2023-06-18T16:28:00.192879Z","iopub.status.idle":"2023-06-18T16:28:00.198709Z","shell.execute_reply.started":"2023-06-18T16:28:00.192851Z","shell.execute_reply":"2023-06-18T16:28:00.197462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Inference is performed (roughly) as follows, ignoring details like how we manage multiple videos:","metadata":{}},{"cell_type":"code","source":"#import tflite_runtime.interpreter as tflite\n#def run_model(model_path):\n    #interpreter = tflite.Interpreter(model_path)\n    #REQUIRED_SIGNATURE = \"serving_default\"\n    #REQUIRED_OUTPUT = \"outputs\"\n    #with open (\"/kaggle/input/fingerspelling-character-map/character_to_prediction_index.json\", \"r\") as f:\n    #character_map = json.load(f)\n    #rev_character_map = {j:i for i,j in character_map.items()}\n    \n    #found_signatures = list(interpreter.get_signature_list().keys())\n    #if REQUIRED_SIGNATURE not in found_signatures:\n        #raise KernelEvalException('Required input signature not found.')\n    #prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n    #output = prediction_fn(inputs=frames)\n    #prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])","metadata":{"execution":{"iopub.status.busy":"2023-06-13T17:27:24.136047Z","iopub.status.idle":"2023-06-13T17:27:24.136717Z","shell.execute_reply.started":"2023-06-13T17:27:24.136373Z","shell.execute_reply":"2023-06-13T17:27:24.136407Z"},"trusted":true},"execution_count":null,"outputs":[]}]}