{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Animated Visualization from Sequence ID\n\nFunction `animate` takes a sequence ID (or list of IDs) and produces an animation of that sequence (or sequences concatenated). Can be configured to show different sets of landmarks, colors, and figure settings.","metadata":{}},{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"markdown","source":"[mediapipe on github](https://github.com/google/mediapipe/tree/master/mediapipe/python/solutions)\n","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"!pip install mediapipe -qq","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T14:59:16.069978Z","iopub.execute_input":"2023-05-23T14:59:16.070364Z","iopub.status.idle":"2023-05-23T14:59:34.337694Z","shell.execute_reply.started":"2023-05-23T14:59:16.07033Z","shell.execute_reply":"2023-05-23T14:59:34.33616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nplt.ioff() # turn off interactive plotting\nfrom matplotlib.animation import FuncAnimation\nimport mediapipe as mp\nfrom types import SimpleNamespace\nfrom IPython.display import HTML\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T14:59:34.345148Z","iopub.execute_input":"2023-05-23T14:59:34.345629Z","iopub.status.idle":"2023-05-23T14:59:46.290349Z","shell.execute_reply.started":"2023-05-23T14:59:34.345577Z","shell.execute_reply":"2023-05-23T14:59:46.289064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = '/kaggle/input/asl-fingerspelling/'\ntrain = pd.read_csv(root_dir+'train.csv')\nsm = pd.read_csv(root_dir+'supplemental_metadata.csv')\ncfg = SimpleNamespace()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T14:59:46.293166Z","iopub.execute_input":"2023-05-23T14:59:46.293903Z","iopub.status.idle":"2023-05-23T14:59:46.605587Z","shell.execute_reply.started":"2023-05-23T14:59:46.293867Z","shell.execute_reply":"2023-05-23T14:59:46.604454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# front-end settings\n\ncfg.parts = { # choose which landmarks to display in animation\n    'right_hand'    : True,\n    'left_hand'     : True,\n    'pose'          : 'arms', # True for all pose landmarks, 'arms' for just arms & shoulders\n    'lips'          : True,\n    'left_eye'      : True,\n    #'left_iris'     : False, # not included in data\n    'left_eyebrow'  : False,\n    'right_eye'     : True,\n    'right_eyebrow' : False,\n    #'right_iris'    : False, # not included in data\n    'face_oval'     : False,\n    'nose'          : False,\n    'contours'      : False,\n    #'irises'        : False, # not included in data\n    'tesselation'   : False,   \n}\nface_features = [\n    'lips', 'left_eye', 'left_iris', 'left_eyebrow',\n    'right_eye', 'right_eyebrow', 'right_iris', 'face_oval',\n    'nose', 'contours', 'irises', 'tesselation',\n]\ncfg.parts['face'] = any([cfg.parts[p] for p in cfg.parts.keys() if p in face_features])\ncfg.colors = { # choose colors for each part\n    'right_hand' : 'firebrick',\n    'left_hand' : 'darkgoldenrod',\n    'pose' : 'steelblue',\n    'face' : 'teal',\n}\ncfg.linewidth=2 # linewidth obvs\ncfg.figsize=(5,5) # figure size obvs\ncfg.xy_lims=(-0.5,1.5) # limits for x and y, same limits for both to preserve aspect ratio\ncfg.fps = 8 # frames per second\ncfg.interval = int(1000/cfg.fps) # delay between frames in milliseconds\ncfg.repeat = False # whether to loop animation (can also just change in viewing pane)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T15:28:42.565687Z","iopub.execute_input":"2023-05-23T15:28:42.566533Z","iopub.status.idle":"2023-05-23T15:28:42.57891Z","shell.execute_reply.started":"2023-05-23T15:28:42.566457Z","shell.execute_reply":"2023-05-23T15:28:42.577256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# back-end settings\n\ncfg.connections = {\n    'right_hand' : mp.solutions.hands_connections.HAND_CONNECTIONS,\n    'left_hand' : mp.solutions.hands_connections.HAND_CONNECTIONS,\n    'pose' : mp.solutions.pose_connections.POSE_CONNECTIONS,\n    'lips' : mp.solutions.face_mesh_connections.FACEMESH_LIPS,\n    'left_eye' : mp.solutions.face_mesh_connections.FACEMESH_LEFT_EYE,\n    #'left_iris' : mp.solutions.face_mesh_connections.FACEMESH_LEFT_IRIS, # not included in data\n    'left_eyebrow' : mp.solutions.face_mesh_connections.FACEMESH_LEFT_EYEBROW,\n    'right_eye' : mp.solutions.face_mesh_connections.FACEMESH_RIGHT_EYE,\n    'right_eyebrow' : mp.solutions.face_mesh_connections.FACEMESH_RIGHT_EYEBROW,\n    #'right_iris' : mp.solutions.face_mesh_connections.FACEMESH_RIGHT_IRIS, # not included in data\n    'face_oval' : mp.solutions.face_mesh_connections.FACEMESH_FACE_OVAL,\n    'nose' : mp.solutions.face_mesh_connections.FACEMESH_NOSE,\n    'contours' : mp.solutions.face_mesh_connections.FACEMESH_CONTOURS,\n    #'irises' : mp.solutions.face_mesh_connections.FACEMESH_IRISES, # not included in data\n    'tesselation' : mp.solutions.face_mesh_connections.FACEMESH_TESSELATION,\n}\ncfg.connections['face'] = frozenset().union(\n    *[cfg.connections[p] for p in cfg.parts.keys() if cfg.parts[p] and p in face_features]\n)\n# manually set pose connections in case of just arms\nif cfg.parts['pose'] == 'arms':\n    cfg.connections['pose'] = frozenset([(11, 12), (11, 13), (13, 15), (12, 14), (14, 16)])\n\ncfg.columns = None # not implemented for now, but possible to only load columns we're using","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T15:28:42.941348Z","iopub.execute_input":"2023-05-23T15:28:42.942374Z","iopub.status.idle":"2023-05-23T15:28:42.953294Z","shell.execute_reply.started":"2023-05-23T15:28:42.942332Z","shell.execute_reply":"2023-05-23T15:28:42.952262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functionality","metadata":{}},{"cell_type":"code","source":"class SequenceDraw:\n    def __init__(self, sid):\n        self.sid = sid\n        \n        pq_path = root_dir + train.loc[train['sequence_id']==sid, 'path'].item()\n        pq = pd.read_parquet(pq_path, columns=cfg.columns) # load parquet file\n        pq = pq.loc[pq.index==sid] # extract the sequence we want\n        self.frames = pq['frame'].values # number of frames\n        self.n_frames = max(self.frames)+1\n        self.lines = {f:[] for f in self.frames} # initialize list of line segments in each frame\n        # actually a dict, not a list, because frames can be non-contiguous e.g. 9 10 11 13 14\n        self.phrase = train.loc[train['sequence_id']==sid, 'phrase'].item() # sample label\n        \n        for frame in self.frames:\n            for part in ['pose', 'face', 'right_hand', 'left_hand']:\n                if not cfg.parts[part]: # if we're not drawing the part, skip it\n                    continue\n                x_cols = pq.columns.str.contains('x_'+part) # get names of x-coord columns\n                x = pq.loc[pq['frame']==frame, x_cols].values[0] # get x-coords\n                y_cols = pq.columns.str.contains('y_'+part) # get names of y-coord columns\n                y = 1 - pq.loc[pq['frame']==frame, y_cols].values[0] # get y-coords\n                # y = 1-y because frames appear to be upside-down\n                for (i, j) in cfg.connections[part]: # add lines to lines\n                    self.lines[frame] += [(x[i], x[j]), (y[i], y[j]), cfg.colors[part]]\n\n    def set_ax(self, ax, progress, position):\n        self.ax = ax\n        self.progress = progress\n        self.position = position\n    \n    def __len__(self):\n        return self.n_frames\n                    \n    def draw(self, frame):\n        self.ax.clear()\n        if frame not in self.lines:\n            return # blank frame if non-contiguous\n        self.ax.plot(*self.lines[frame], linewidth=cfg.linewidth)\n        self.ax.set_xlim(cfg.xy_lims)\n        self.ax.set_ylim(cfg.xy_lims)\n        self.ax.set_title(f'sid {self.sid}:    {self.phrase}')\n        self.ax.set_xlabel(f'seq {self.position}    frame {frame}/{self.n_frames}')\n        if self.progress:\n            self.progress.update()\n\nclass Animation:\n    def __init__(self, sids):\n        try: # make sure it's an iterable\n            iter(sids)\n        except TypeError:\n            sids = [sids]\n        self.seqs = []\n        for sid in sids:\n            self.seqs += [SequenceDraw(sid)]\n        self.total_frames = sum([len(s) for s in self.seqs])\n        \n        self.fig, ax = plt.subplots(figsize=cfg.figsize)\n        progress = tqdm(total=self.total_frames, position=0, leave=True)\n        for i, seq in enumerate(self.seqs):\n            seq.set_ax(ax, progress, f\"{i+1}/{len(self.seqs)}\")\n        \n    def draw(self, i):\n        j = 0\n        while i >= len(self.seqs[j]):\n            i -= len(self.seqs[j])\n            j += 1\n        self.seqs[j].draw(i)\n    \n    def animate(self):\n        anim = FuncAnimation(\n            fig=self.fig,\n            func=self.draw,\n            frames=self.total_frames,\n            interval=cfg.interval,\n            repeat=cfg.repeat,\n        )\n        display(HTML(anim.to_jshtml()))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T15:28:45.426416Z","iopub.execute_input":"2023-05-23T15:28:45.427556Z","iopub.status.idle":"2023-05-23T15:28:45.451469Z","shell.execute_reply.started":"2023-05-23T15:28:45.427513Z","shell.execute_reply":"2023-05-23T15:28:45.450493Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Animations","metadata":{}},{"cell_type":"code","source":"\"\"\" Interesting sequences to visualize \"\"\"\nsid = train['sequence_id'].values[52] # ordinary \"bad\" sample, 6 frames mostly without hands for label \"106-703-4104\"\nsid = train['sequence_id'].values[53] # non-contiguous set of frames: ... 9 10 11 13 14 ...\nsid = train['sequence_id'].values[54] # ordinary \"good\" sample, 140 frames mostly with hands for label \"tomeka salinas\"\nsid = 1450258177 # max number of frames at 807, also max frames per char at 67.25, label \"5190386.aspx\"\nsid = 633260881 # max number of frames containing a hand at 598\nsid = 1788389712 # sequence with 25 RH frames, 30 LH frames, pid=20---RH turns out to be the relevant one\nsid = 70258272 # sequence with 154 RH frames, 160 LH frames, pid=20---RH turns out to be the relevant one\nsid = 377333951 # sequence with 6 RH frames, 6 LH frames, pid=72---mediapipe getting hands confused?\nsid = 1820769464 # ordinary sequence with pid=72","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:59:46.694894Z","iopub.execute_input":"2023-05-23T14:59:46.695354Z","iopub.status.idle":"2023-05-23T14:59:46.726534Z","shell.execute_reply.started":"2023-05-23T14:59:46.695314Z","shell.execute_reply":"2023-05-23T14:59:46.725062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sid = train['sequence_id'].values[52:55]\nanim = Animation(sid)\nanim.animate() # can take a minute or two for longer sequences","metadata":{"execution":{"iopub.status.busy":"2023-05-23T15:50:16.865958Z","iopub.execute_input":"2023-05-23T15:50:16.866442Z","iopub.status.idle":"2023-05-23T15:54:20.504188Z","shell.execute_reply.started":"2023-05-23T15:50:16.866405Z","shell.execute_reply":"2023-05-23T15:54:20.503156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}