{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import dependencies\nNote: Modifed to supplemental_landmarks","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd, math\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\nimport matplotlib.animation as animation\nfrom matplotlib.animation import FuncAnimation\n\nplt.rcParams[\"animation.html\"] = \"jshtml\"\nplt.rcParams['figure.dpi'] = 80\nplt.ioff()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-04T08:56:52.826544Z","iopub.execute_input":"2023-08-04T08:56:52.82694Z","iopub.status.idle":"2023-08-04T08:56:52.835718Z","shell.execute_reply.started":"2023-08-04T08:56:52.826909Z","shell.execute_reply":"2023-08-04T08:56:52.834623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Demo gif\n\n<img src=\"https://i.imgur.com/eQOH82t.gif\">","metadata":{}},{"cell_type":"markdown","source":"# Initialize labels and matplot","metadata":{}},{"cell_type":"code","source":"INPUT_DIR = \"/kaggle/input/asl-fingerspelling/\"\n\nLIP_LIST = [0,17,37,39,40,57,84,91,181,267,269,270,287,314,321,405]\n\nPOSE_LIST = list(range(11, 21))\nRHAND_POSE_LIST = [12, 14, 16, 18, 20, 22]\nLHAND_POSE_LIST = [11, 13, 15, 17, 19, 21]\n\nRHAND_LBLS = [f'x_right_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)] + [f'z_right_hand_{i}' for i in range(21)]\nLHAND_LBLS = [f'x_left_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)]\n\nLIP_LBLS_X   = [f'x_face_{i}' for i in LIP_LIST] \nLIP_LBLS_Y   = [f'y_face_{i}' for i in LIP_LIST] \nLIP_LBLS_Z   = [f'z_face_{i}' for i in LIP_LIST] \n\nRHAND_POSE_LBLS_X   = [f'x_pose_{i}' for i in RHAND_POSE_LIST] \nRHAND_POSE_LBLS_Y   = [f'y_pose_{i}' for i in RHAND_POSE_LIST] \nRHAND_POSE_LBLS_Z   = [f'z_pose_{i}' for i in RHAND_POSE_LIST] \n\nLHAND_POSE_LBLS_X   = [f'x_pose_{i}' for i in LHAND_POSE_LIST] \nLHAND_POSE_LBLS_Y   = [f'y_pose_{i}' for i in LHAND_POSE_LIST] \nLHAND_POSE_LBLS_Z   = [f'z_pose_{i}' for i in LHAND_POSE_LIST] \n\nLIP_LBLS     = LIP_LBLS_X + LIP_LBLS_Y + LIP_LBLS_Z\n\nRHAND_POSE_LBLS     = RHAND_POSE_LBLS_X + RHAND_POSE_LBLS_Y + RHAND_POSE_LBLS_Z\nLHAND_POSE_LBLS     = LHAND_POSE_LBLS_X + LHAND_POSE_LBLS_Y + LHAND_POSE_LBLS_Z\n\nPOSE_LBLS = RHAND_POSE_LBLS + LHAND_POSE_LBLS\n\nfig = plt.figure(figsize=(15, 5))\nrh_ax = fig.add_subplot(131, projection=\"3d\")\nrh_ax.view_init(100, 60)\n\nlh_ax = fig.add_subplot(133, projection=\"3d\")\nlh_ax.view_init(100, 60)\n\nlip_ax = fig.add_subplot(132, projection=\"3d\")\nlip_ax.view_init(300, 200)\n\nfig.tight_layout()\nfig.subplots_adjust(left=0, right=1, bottom=0, top=0.8)\n\nLINE_WIDTH = 5\n\ntrain_df = pd.read_csv(f'{INPUT_DIR}supplemental_metadata.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-04T08:56:52.841948Z","iopub.execute_input":"2023-08-04T08:56:52.842427Z","iopub.status.idle":"2023-08-04T08:56:53.189767Z","shell.execute_reply.started":"2023-08-04T08:56:52.842393Z","shell.execute_reply":"2023-08-04T08:56:53.188709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions\nThe function **get_random_sequence** that takes an optional argument phrase_idx. If phrase_idx is not provided, it generates a random integer within the range of the length of a DataFrame called train_df. It then selects a row from train_df based on the generated or provided phrase_idx. It extracts some values from the selected row and uses them to construct a file path. It reads a Parquet file at that path into a DataFrame called dataset. Finally, it converts dataset to a pandas DataFrame called sequence_df and returns phrase_idx, phrase, and sequence_df.","metadata":{}},{"cell_type":"code","source":"def get_random_sequence(phrase_idx=None):\n    if phrase_idx == None:\n        phrase_idx = np.random.randint(len(train_df))\n\n    selected_row = train_df.iloc[phrase_idx]\n    file_id = int(selected_row['file_id'])\n    sequence_id = int(selected_row['sequence_id'])\n    phrase = selected_row['phrase']\n       \n    dataset = pq.read_table(\n        f'{INPUT_DIR}supplemental_landmarks/{str(file_id)}.parquet',\n        filters=[\n            [('sequence_id', '=', sequence_id)],\n        ],\n        columns= LHAND_LBLS+RHAND_LBLS+LIP_LBLS+POSE_LBLS\n    )\n    \n    sequence_df = dataset.to_pandas()\n\n    return phrase_idx, phrase, sequence_df","metadata":{"execution":{"iopub.status.busy":"2023-08-04T08:56:53.193734Z","iopub.execute_input":"2023-08-04T08:56:53.194075Z","iopub.status.idle":"2023-08-04T08:56:53.201643Z","shell.execute_reply.started":"2023-08-04T08:56:53.194049Z","shell.execute_reply":"2023-08-04T08:56:53.200409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This code defines a function **hand_lines** that takes in a list of hand landmarks. It then extracts the x, y, and z coordinates for different parts of the hand (palm, thumb, index finger, middle finger, ring finger, and pinky) from the input list. The extracted coordinates are returned as lists of x, y, and z values for each hand part.","metadata":{}},{"cell_type":"code","source":"def hand_lines(hands_landmarks):\n    palm_list = [0, 1, 5, 9, 13, 17, 0]\n    thumb_list = [1, 2, 3, 4]\n    index_finger_list = [5, 6, 7, 8]\n    middle_finger_list = [9, 10, 11, 12]\n    ring_finger_list = [13, 14, 15, 16]\n    pinky_list = [17, 18, 19, 20]\n\n    palm_x, palm_y, palm_z = zip(*[hands_landmarks[i] for i in palm_list])\n    thumb_x, thumb_y, thumb_z = zip(*[hands_landmarks[i] for i in thumb_list])\n    index_finger_x, index_finger_y, index_finger_z = zip(*[hands_landmarks[i] for i in index_finger_list])\n    middle_finger_x, middle_finger_y, middle_finger_z = zip(*[hands_landmarks[i] for i in middle_finger_list])\n    ring_finger_x, ring_finger_y, ring_finger_z = zip(*[hands_landmarks[i] for i in ring_finger_list])\n    pinky_x, pinky_y, pinky_z = zip(*[hands_landmarks[i] for i in pinky_list])\n\n    return [list(palm_x), list(palm_y), list(palm_z)], [list(thumb_x), list(thumb_y), list(thumb_z)], [list(index_finger_x), list(index_finger_y), list(index_finger_z)], [list(middle_finger_x), list(middle_finger_y), list(middle_finger_z)], [list(ring_finger_x), list(ring_finger_y), list(ring_finger_z)], [list(pinky_x), list(pinky_y), list(pinky_z)]\n","metadata":{"execution":{"iopub.status.busy":"2023-08-04T08:56:53.203627Z","iopub.execute_input":"2023-08-04T08:56:53.204538Z","iopub.status.idle":"2023-08-04T08:56:53.217096Z","shell.execute_reply.started":"2023-08-04T08:56:53.204504Z","shell.execute_reply":"2023-08-04T08:56:53.216038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This code defines a function **lip_lines** that takes a list of lip landmarks as input. It creates two lists, up_list and down_list, which contain indices of lip landmarks. It then creates a dictionary lip_point by pairing each index in LIP_LIST with the corresponding lip landmark from the input. The code then extracts the x, y, and z coordinates of the lip landmarks specified by up_list and down_list using list comprehensions and the zip function. Finally, it returns two lists, each containing the x, y, and z coordinates of the lip landmarks specified by up_list and down_list.","metadata":{}},{"cell_type":"code","source":"def lip_lines(lip_landmarks):\n    up_list = [57, 40, 39, 37, 0, 267, 269, 270, 287]\n    down_list = [57, 91, 181, 84, 17, 314, 405, 321, 287]\n\n    lip_point =  {LIP_LIST: lip_landmarks for LIP_LIST, lip_landmarks in zip(LIP_LIST, lip_landmarks)}\n\n    up_lip_x, up_lip_y, up_lip_z = zip(*[lip_point[idx] for idx in up_list])\n    down_lip_x, down_lip_y, down_lip_z = zip(*[lip_point[idx] for idx in down_list])\n\n    return [list(up_lip_x), list(up_lip_y), list(up_lip_z)], [list(down_lip_x), list(down_lip_y), list(down_lip_z)]","metadata":{"execution":{"iopub.status.busy":"2023-08-04T08:56:53.219979Z","iopub.execute_input":"2023-08-04T08:56:53.220707Z","iopub.status.idle":"2023-08-04T08:56:53.228074Z","shell.execute_reply.started":"2023-08-04T08:56:53.220677Z","shell.execute_reply":"2023-08-04T08:56:53.227293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The function **pose_lines** that takes in a list of hand pose_landmarks as an argument. It creates three new lists, pose_x, pose_y, and pose_z, by extracting the first, second, and third elements from each sublist in pose_landmarks. It then appends the third element from each list to itself. Finally, it returns a list containing pose_x, pose_y, and pose_z.","metadata":{}},{"cell_type":"code","source":"def pose_lines(pose_landmarks):\n    pose_x = [pose_landmarks[idx][0] for idx in range(len(pose_landmarks))]\n    pose_y = [pose_landmarks[idx][1] for idx in range(len(pose_landmarks))]\n    pose_z = [pose_landmarks[idx][2] for idx in range(len(pose_landmarks))]\n    pose_x.append(pose_x[2])\n    pose_y.append(pose_y[2])\n    pose_z.append(pose_z[2])\n    return [pose_x, pose_y, pose_z]","metadata":{"execution":{"iopub.status.busy":"2023-08-04T08:56:53.229524Z","iopub.execute_input":"2023-08-04T08:56:53.230224Z","iopub.status.idle":"2023-08-04T08:56:53.241938Z","shell.execute_reply.started":"2023-08-04T08:56:53.230191Z","shell.execute_reply":"2023-08-04T08:56:53.241222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This code takes a list of pose landmarks and extracts the x, y, and z coordinates into separate lists. It then appends the values at index 2 of each list to the end of the respective list. Finally, it returns a list containing the three separate lists of x, y, and z coordinates.","metadata":{}},{"cell_type":"code","source":"def plot_landmarks(plt, ax_list, landmarks, hdlbl, nan_count, phrase, chr, idx, totfr, cfr):\n    ax_list[0].cla()\n    ax_list[1].cla()\n    ax_list[2].cla()\n\n    handedness_index = 0\n\n    palm, thumb, index_finger, middle_finger, ring_finger, pinky = hand_lines(landmarks[0])\n\n    ax_list[handedness_index].plot(palm[0], palm[1], palm[2], linewidth=LINE_WIDTH)\n    ax_list[handedness_index].plot(thumb[0], thumb[1], thumb[2], linewidth=LINE_WIDTH*2)\n    ax_list[handedness_index].plot(index_finger[0], index_finger[1], index_finger[2], linewidth=LINE_WIDTH*1.5)\n    ax_list[handedness_index].plot(middle_finger[0], middle_finger[1], middle_finger[2], linewidth=LINE_WIDTH*1.4)\n    ax_list[handedness_index].plot(ring_finger[0], ring_finger[1], ring_finger[2], linewidth=LINE_WIDTH*1.3)\n    ax_list[handedness_index].plot(pinky[0], pinky[1], pinky[2], linewidth=LINE_WIDTH*1.2)\n    ax_list[handedness_index].set_title(hdlbl+\" Fingers, Index: \"+str(idx)+\", Chr: \"+chr, pad=20)\n\n    handedness_index = 1\n\n    pose = pose_lines(landmarks[1])\n    \n    ax_list[handedness_index].plot(pose[0], pose[1], pose[2], linewidth=LINE_WIDTH*1.2)\n    ax_list[handedness_index].set_title(hdlbl+\" Hand, Frame: \"+str(cfr)+\"/\"+str(totfr), pad=20)\n\n    handedness_index = 2\n\n    up_lip, down_lip = lip_lines(landmarks[2])\n    \n    ax_list[handedness_index].plot(up_lip[0], up_lip[1], up_lip[2], linewidth=LINE_WIDTH*2)\n    ax_list[handedness_index].plot(down_lip[0], down_lip[1], down_lip[2], linewidth=LINE_WIDTH*4)\n    ax_list[handedness_index].set_title(\"Phrase: \"+ phrase+\", NaNs: \"+str(int(nan_count/21))+\"/\"+str(totfr), pad=20)\n    \n    return","metadata":{"execution":{"iopub.status.busy":"2023-08-04T08:56:53.243481Z","iopub.execute_input":"2023-08-04T08:56:53.244378Z","iopub.status.idle":"2023-08-04T08:56:53.258216Z","shell.execute_reply.started":"2023-08-04T08:56:53.244315Z","shell.execute_reply":"2023-08-04T08:56:53.257038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This code snippet defines a function called filter_df that takes a DataFrame called seq_df as input.\n\nThe function iterates over the rows of seq_df and filters specific columns based on regular expressions. It then appends the filtered values to different lists.\n\nAfter iterating through all the rows, the function checks which hand (right or left) has more NaN values and sets the hand_landmarks and pose_landmarks variables accordingly. It also sets the htype variable to indicate whether it's the right hand or the left hand with more NaN values.\n\nFinally, the function returns a list containing the hand_landmarks, pose_landmarks, and lip_landmarks lists, as well as the htype and nan variables.","metadata":{}},{"cell_type":"code","source":"def filter_df(seq_df):\n\n    right_hand_landmarks = []\n    left_hand_landmarks = []\n    right_pose_landmarks = []\n    left_pose_landmarks = []    \n    lip_landmarks =[]\n    rnan = 0\n    lnan = 0\n    htype= \"Right\"\n    for seq_idx in range(len(seq_df)):\n        r_x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_right_hand.*\").values\n        r_y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_right_hand.*\").values\n        r_z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_right_hand.*\").values\n\n        l_x_pose = seq_df.iloc[seq_idx].filter(regex=\"x_left_hand.*\").values\n        l_y_pose = seq_df.iloc[seq_idx].filter(regex=\"y_left_hand.*\").values\n        l_z_pose = seq_df.iloc[seq_idx].filter(regex=\"z_left_hand.*\").values\n\n        lip_x_pose = seq_df.iloc[seq_idx].filter(LIP_LBLS_X).values\n        lip_y_pose = seq_df.iloc[seq_idx].filter(LIP_LBLS_Y).values\n        lip_z_pose = seq_df.iloc[seq_idx].filter(LIP_LBLS_Z).values\n\n        rh_x_pose = seq_df.iloc[seq_idx].filter(RHAND_POSE_LBLS_X).values\n        rh_y_pose = seq_df.iloc[seq_idx].filter(RHAND_POSE_LBLS_Y).values\n        rh_z_pose = seq_df.iloc[seq_idx].filter(RHAND_POSE_LBLS_Z).values\n        \n        lh_x_pose = seq_df.iloc[seq_idx].filter(LHAND_POSE_LBLS_X).values\n        lh_y_pose = seq_df.iloc[seq_idx].filter(LHAND_POSE_LBLS_Y).values\n        lh_z_pose = seq_df.iloc[seq_idx].filter(LHAND_POSE_LBLS_Z).values\n\n        right_hand_landmark = []\n        left_hand_landmark = []\n        right_pose_landmark = []\n        left_pose_landmark = []\n        lip_landmark = []\n\n        for x, y, z in zip(r_x_pose, r_y_pose, r_z_pose):\n            right_hand_landmark.append([x, y, z])\n            if math.isnan(x) : rnan +=1\n        right_hand_landmarks.append(right_hand_landmark) \n\n        for x, y, z in zip(l_x_pose, l_y_pose, l_z_pose):\n            left_hand_landmark.append([1-x, y, z])\n            if math.isnan(x) : lnan +=1\n        left_hand_landmarks.append(left_hand_landmark) \n\n        for x, y, z in zip(rh_x_pose, rh_y_pose, rh_z_pose):\n            right_pose_landmark.append([x, y, z])\n        right_pose_landmarks.append(right_pose_landmark) \n\n        for x, y, z in zip(lh_x_pose, lh_y_pose, lh_z_pose):\n            left_pose_landmark.append([1-x, y, z])\n        left_pose_landmarks.append(left_pose_landmark) \n\n        for x, y, z in zip(lip_x_pose, lip_y_pose,lip_z_pose):\n            lip_landmark.append([x, y, z])\n        lip_landmarks.append(lip_landmark) \n\n        if lnan > rnan:\n            hand_landmarks = right_hand_landmarks\n            pose_landmarks = right_pose_landmarks\n            nan = rnan\n        else:\n            hand_landmarks = left_hand_landmarks\n            pose_landmarks = left_pose_landmarks\n            htype= \"Left\"\n            nan = lnan\n\n    return  [hand_landmarks, pose_landmarks, lip_landmarks], htype, nan","metadata":{"execution":{"iopub.status.busy":"2023-08-04T08:56:53.260866Z","iopub.execute_input":"2023-08-04T08:56:53.261518Z","iopub.status.idle":"2023-08-04T08:56:53.277768Z","shell.execute_reply.started":"2023-08-04T08:56:53.261486Z","shell.execute_reply":"2023-08-04T08:56:53.277101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The function getchr that takes three parameters: phrase, i, and totlen to predict character for the current frame. It calculates the length of the phrase and then calculates the fraction of the total length (totlen) that corresponds to the current index (i). It uses this fraction to determine the index (current_chr_idx) of the character to return from the phrase. If the calculated index is greater than or equal to the length of the phrase, it sets the index to the last character in the phrase. The function returns the selected character and the index.","metadata":{}},{"cell_type":"code","source":"def getchr(phrase, i, totlen):\n    plen = len(phrase)\n    current_chr_idx = math.floor((plen/totlen)*i)\n    \n    if current_chr_idx >= plen-1: current_chr_idx = plen-1\n\n    return phrase[current_chr_idx], current_chr_idx\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-04T09:05:45.061253Z","iopub.execute_input":"2023-08-04T09:05:45.061624Z","iopub.status.idle":"2023-08-04T09:05:45.067244Z","shell.execute_reply.started":"2023-08-04T09:05:45.061588Z","shell.execute_reply":"2023-08-04T09:05:45.066253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run\n\nWait for 3mins","metadata":{}},{"cell_type":"code","source":"\nid, phrase, df = get_random_sequence()\n\nprint(f\"Phrase: {phrase}\")\n\ndata_frames, hdlbl, nan = filter_df(df)\ntotframe = len(data_frames[0])\n\n# for i in range(0, len(data_frames[0])):\n#     plot_landmarks(plt, [rh_ax, lh_ax, lip_ax], [data_frames[0][i],data_frames[1][i],data_frames[2][i]])\n\ndef animate(i):\n    chr, idx = getchr(phrase, i, totframe)\n    \n    plot_landmarks(plt, [rh_ax, lh_ax, lip_ax], [data_frames[0][i], data_frames[1][i], data_frames[2][i]], hdlbl, nan, phrase, chr, idx, totframe, i)\n\n# myAnimation = animation.FuncAnimation(fig, animate, len(data_frames[0]), interval=20, blit=False)\n# myAnimation.save(\"output.gif\", writer='imagemagick', fps=60)\n\nFuncAnimation(fig, animate, frames=totframe)","metadata":{"execution":{"iopub.status.busy":"2023-08-04T09:07:49.451294Z","iopub.execute_input":"2023-08-04T09:07:49.451656Z","iopub.status.idle":"2023-08-04T09:09:06.713381Z","shell.execute_reply.started":"2023-08-04T09:07:49.451627Z","shell.execute_reply":"2023-08-04T09:09:06.709219Z"},"trusted":true},"execution_count":null,"outputs":[]}]}