{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nimport plotly.graph_objects as go\nimport plotly.io as pio\npio.templates.default = \"simple_white\"","metadata":{"id":"ZTszvA590pOD","execution":{"iopub.status.busy":"2023-10-19T08:49:13.55878Z","iopub.execute_input":"2023-10-19T08:49:13.559208Z","iopub.status.idle":"2023-10-19T08:49:13.587988Z","shell.execute_reply.started":"2023-10-19T08:49:13.559174Z","shell.execute_reply":"2023-10-19T08:49:13.586727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_new_to_old_style(sequence):\n    types = []\n    landmark_indexes = []\n    for column in list(sequence.columns)[1:544]:\n        parts = column.split(\"_\")\n        if len(parts) == 4:\n            types.append(parts[1] + \"_\" + parts[2])\n        else:\n            types.append(parts[1])\n\n        landmark_indexes.append(int(parts[-1]))\n\n    data = {\n        \"frame\": [],\n        \"type\": [],\n        \"landmark_index\": [],\n        \"x\": [],\n        \"y\": [],\n        \"z\": []\n    }\n\n    for index, row in sequence.iterrows():\n        data[\"frame\"] += [int(row.frame)]*543\n        data[\"type\"] += types\n        data[\"landmark_index\"] += landmark_indexes\n\n        for _type, landmark_index in zip(types, landmark_indexes):\n            data[\"x\"].append(row[f\"x_{_type}_{landmark_index}\"])\n            data[\"y\"].append(row[f\"y_{_type}_{landmark_index}\"])\n            data[\"z\"].append(row[f\"z_{_type}_{landmark_index}\"])\n\n    return pd.DataFrame.from_dict(data)\n\n# assign desired colors to landmarks\ndef assign_color(row):\n    if row == 'face':\n        return 'red'\n    elif 'hand' in row:\n        return 'dodgerblue'\n    else:\n        return 'green'\n\n# specifies the plotting order\ndef assign_order(row):\n    if row.type == 'face':\n        return row.landmark_index + 101\n    elif row.type == 'pose':\n        return row.landmark_index + 30\n    elif row.type == 'left_hand':\n        return row.landmark_index + 80\n    else:\n        return row.landmark_index\n\ndef visualise2d_landmarks(parquet_df, title=\"\"):\n    connections = [\n        [0, 1, 2, 3, 4,],\n        [0, 5, 6, 7, 8],\n        [0, 9, 10, 11, 12],\n        [0, 13, 14, 15, 16],\n        [0, 17, 18, 19, 20],\n\n\n        [38, 36, 35, 34, 30, 31, 32, 33, 37],\n        [40, 39],\n        [52, 46, 50, 48, 46, 44, 42, 41, 43, 45, 47, 49, 45, 51],\n        [42, 54, 56, 58, 60, 62, 58],\n        [41, 53, 55, 57, 59, 61, 57],\n        [54, 53],\n\n\n        [80, 81, 82, 83, 84, ],\n        [80, 85, 86, 87, 88],\n        [80, 89, 90, 91, 92],\n        [80, 93, 94, 95, 96],\n        [80, 97, 98, 99, 100], ]\n\n    parquet_df = map_new_to_old_style(parquet_df)\n    frames = sorted(set(parquet_df.frame))\n    first_frame = min(frames)\n    parquet_df['color'] = parquet_df.type.apply(lambda row: assign_color(row))\n    parquet_df['plot_order'] = parquet_df.apply(lambda row: assign_order(row), axis=1)\n    first_frame_df = parquet_df[parquet_df.frame == first_frame].copy()\n    first_frame_df = first_frame_df.sort_values([\"plot_order\"]).set_index('plot_order')\n\n\n    frames_l = []\n    for frame in frames:\n        filtered_df = parquet_df[parquet_df.frame == frame].copy()\n        filtered_df = filtered_df.sort_values([\"plot_order\"]).set_index(\"plot_order\")\n        traces = [go.Scatter(\n            x=filtered_df['x'],\n            y=filtered_df['y'],\n            mode='markers',\n            marker=dict(\n                color=filtered_df.color,\n                size=9))]\n\n        for i, seg in enumerate(connections):\n            trace = go.Scatter(\n                    x=filtered_df.loc[seg]['x'],\n                    y=filtered_df.loc[seg]['y'],\n                    mode='lines',\n            )\n            traces.append(trace)\n        frame_data = go.Frame(data=traces, traces = [i for i in range(17)])\n        frames_l.append(frame_data)\n\n    traces = [go.Scatter(\n        x=first_frame_df['x'],\n        y=first_frame_df['y'],\n        mode='markers',\n        marker=dict(\n            color=first_frame_df.color,\n            size=9\n        )\n    )]\n    for i, seg in enumerate(connections):\n        trace = go.Scatter(\n            x=first_frame_df.loc[seg]['x'],\n            y=first_frame_df.loc[seg]['y'],\n            mode='lines',\n            line=dict(\n                color='black',\n                width=2\n            )\n        )\n        traces.append(trace)\n    fig = go.Figure(\n        data=traces,\n        frames=frames_l\n    )\n\n\n    fig.update_layout(\n        width=500,\n        height=800,\n        scene={\n            'aspectmode': 'data',\n        },\n        updatemenus=[\n            {\n                \"buttons\": [\n                    {\n                        \"args\": [None, {\"frame\": {\"duration\": 100,\n                                                  \"redraw\": True},\n                                        \"fromcurrent\": True,\n                                        \"transition\": {\"duration\": 0}}],\n                        \"label\": \"&#9654;\",\n                        \"method\": \"animate\",\n                    },\n\n                ],\n                \"direction\": \"left\",\n                \"pad\": {\"r\": 100, \"t\": 100},\n                \"font\": {\"size\":30},\n                \"type\": \"buttons\",\n                \"x\": 0.1,\n                \"y\": 0,\n            }\n        ],\n    )\n    camera = dict(\n        up=dict(x=0, y=-1, z=0),\n        eye=dict(x=0, y=0, z=2.5)\n    )\n    fig.update_layout(title_text=title, title_x=0.5)\n    fig.update_layout(scene_camera=camera, showlegend=False)\n    fig.update_layout(xaxis = dict(visible=False),\n            yaxis = dict(visible=False),\n    )\n    fig.update_yaxes(autorange=\"reversed\")\n\n    fig.show()\n\n\ndef get_phrase(df, file_id, sequence_id):\n    return df[\n        np.logical_and(\n            df.file_id == file_id,\n            df.sequence_id == sequence_id\n        )\n    ].phrase.iloc[0]","metadata":{"id":"93sgvtjZNQNe","execution":{"iopub.status.busy":"2023-10-19T08:49:13.686714Z","iopub.execute_input":"2023-10-19T08:49:13.687141Z","iopub.status.idle":"2023-10-19T08:49:13.714223Z","shell.execute_reply.started":"2023-10-19T08:49:13.687109Z","shell.execute_reply":"2023-10-19T08:49:13.713144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\nsupplemental_df = pd.read_csv('/kaggle/input/asl-fingerspelling/supplemental_metadata.csv')\npd.set_option('display.max_columns', None)\nsupplemental_df.head(10)","metadata":{"id":"TYn9m1rV1PoV","outputId":"3915e623-7b02-4450-8041-72a79f486a1c","execution":{"iopub.status.busy":"2023-10-19T08:49:13.716061Z","iopub.execute_input":"2023-10-19T08:49:13.716358Z","iopub.status.idle":"2023-10-19T08:49:14.076552Z","shell.execute_reply.started":"2023-10-19T08:49:13.716334Z","shell.execute_reply":"2023-10-19T08:49:14.075254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_df)","metadata":{"id":"s-AlJUNSmeGW","outputId":"b2c47ad8-5248-4b19-c0f9-fa3fb75ad6dc","execution":{"iopub.status.busy":"2023-10-19T08:49:14.077852Z","iopub.execute_input":"2023-10-19T08:49:14.078228Z","iopub.status.idle":"2023-10-19T08:49:14.093864Z","shell.execute_reply.started":"2023-10-19T08:49:14.078196Z","shell.execute_reply":"2023-10-19T08:49:14.092631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_feature_data(data, feature,return_df=False):\n  feature_count = data[feature]\n  print(f'number of {feature}  ', len(feature_count))\n  unique_feature = data[feature].unique()\n  print(f'number of {feature} unique: ',len(unique_feature))\n  if return_df == 1:\n    feature_counts = data[feature].value_counts()\n    feature_data = pd.DataFrame({ feature: feature_counts.index, feature+'_count': feature_counts.values})\n    return feature_data","metadata":{"id":"Yu8cqhpESS-s","execution":{"iopub.status.busy":"2023-10-19T08:49:14.0965Z","iopub.execute_input":"2023-10-19T08:49:14.096868Z","iopub.status.idle":"2023-10-19T08:49:14.105653Z","shell.execute_reply.started":"2023-10-19T08:49:14.096837Z","shell.execute_reply":"2023-10-19T08:49:14.104395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_feature_data(supplemental_df,'phrase',return_df=True)","metadata":{"id":"WBboXk__Thty","outputId":"f114a97d-ad94-4920-debb-a43bfd3d9ff6","execution":{"iopub.status.busy":"2023-10-19T08:49:14.106728Z","iopub.execute_input":"2023-10-19T08:49:14.107102Z","iopub.status.idle":"2023-10-19T08:49:14.145773Z","shell.execute_reply.started":"2023-10-19T08:49:14.107071Z","shell.execute_reply":"2023-10-19T08:49:14.144784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_feature_data(train_df,'phrase',return_df=True)","metadata":{"id":"WdWXQHveTh25","outputId":"7d472ab8-a7f3-4ed9-cb48-09d9cc02c67e","execution":{"iopub.status.busy":"2023-10-19T08:49:14.147709Z","iopub.execute_input":"2023-10-19T08:49:14.148149Z","iopub.status.idle":"2023-10-19T08:49:14.205149Z","shell.execute_reply.started":"2023-10-19T08:49:14.148084Z","shell.execute_reply":"2023-10-19T08:49:14.204068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supplemental_df[supplemental_df['phrase']=='coming up with killer sound bites']","metadata":{"id":"m52zlSTTTLMD","outputId":"1a079e13-96b0-48a1-92c9-6a80f659ad6f","execution":{"iopub.status.busy":"2023-10-19T08:49:14.206796Z","iopub.execute_input":"2023-10-19T08:49:14.207236Z","iopub.status.idle":"2023-10-19T08:49:14.228672Z","shell.execute_reply.started":"2023-10-19T08:49:14.207194Z","shell.execute_reply":"2023-10-19T08:49:14.227187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['file_id']==5414471]","metadata":{"id":"lVY2M4MYJI26","outputId":"3ef78ca0-4802-480f-b557-56d79cfebfb4","execution":{"iopub.status.busy":"2023-10-19T08:49:14.230507Z","iopub.execute_input":"2023-10-19T08:49:14.231012Z","iopub.status.idle":"2023-10-19T08:49:14.25273Z","shell.execute_reply.started":"2023-10-19T08:49:14.23095Z","shell.execute_reply":"2023-10-19T08:49:14.251483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_feature_data(train_df,'file_id',return_df=True)","metadata":{"id":"SXdGIo5C7h5N","outputId":"229b67dc-326d-4754-b0e1-79828ee781a2","execution":{"iopub.status.busy":"2023-10-19T08:49:14.25402Z","iopub.execute_input":"2023-10-19T08:49:14.254354Z","iopub.status.idle":"2023-10-19T08:49:14.270029Z","shell.execute_reply.started":"2023-10-19T08:49:14.254326Z","shell.execute_reply":"2023-10-19T08:49:14.268796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Let's try to visualize sequence","metadata":{"id":"_FoW2t6a6Ael"}},{"cell_type":"code","source":"sequence_id = 1817123330\nfile_id = 5414471","metadata":{"id":"L32LT7bo2Ioy","execution":{"iopub.status.busy":"2023-10-19T08:49:14.272777Z","iopub.execute_input":"2023-10-19T08:49:14.273155Z","iopub.status.idle":"2023-10-19T08:49:14.278944Z","shell.execute_reply.started":"2023-10-19T08:49:14.273126Z","shell.execute_reply":"2023-10-19T08:49:14.277535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_sign = f\"/kaggle/input/asl-fingerspelling/train_landmarks/{file_id}.parquet\"\nsign = pd.read_parquet(path_to_sign)","metadata":{"id":"ho6S3lM66F9N","execution":{"iopub.status.busy":"2023-10-19T08:49:14.280494Z","iopub.execute_input":"2023-10-19T08:49:14.281519Z","iopub.status.idle":"2023-10-19T08:49:24.32534Z","shell.execute_reply.started":"2023-10-19T08:49:14.281482Z","shell.execute_reply":"2023-10-19T08:49:24.323536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign","metadata":{"id":"_u2CSKcOJx-4","outputId":"8a39f397-cad0-45d8-f372-83a43d60e26d","execution":{"iopub.status.busy":"2023-10-19T08:49:24.327465Z","iopub.execute_input":"2023-10-19T08:49:24.328189Z","iopub.status.idle":"2023-10-19T08:49:25.628638Z","shell.execute_reply.started":"2023-10-19T08:49:24.328149Z","shell.execute_reply":"2023-10-19T08:49:25.627492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Face Landmarks (Khuôn mặt): 468 điểm đặc trưng.\nLeft Hand Landmarks (Tay trái): 21 điểm đặc trưng.\nPose Landmarks (Tư thế cơ thể): 33 điểm đặc trưng.\nRight Hand Landmarks (Tay phải): 21 điểm đặc trưng.\n468 (face) + 21 (left hand) + 33 (pose) + 21 (right hand) = 543 *3(x,y,z) = 1629\n","metadata":{"id":"JbdH6EcJKQF1"}},{"cell_type":"code","source":"np.unique(sign.index)","metadata":{"id":"5DInhe-OJyKE","outputId":"c5fb6a5a-c33d-42c3-e078-8df26814f0ea","execution":{"iopub.status.busy":"2023-10-19T08:49:25.629905Z","iopub.execute_input":"2023-10-19T08:49:25.631107Z","iopub.status.idle":"2023-10-19T08:49:25.648819Z","shell.execute_reply.started":"2023-10-19T08:49:25.631044Z","shell.execute_reply":"2023-10-19T08:49:25.647594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence = sign[sign.index == sequence_id]\nsequence","metadata":{"id":"hk0hqN0qKjgV","outputId":"230b2cf9-0870-404d-d2ca-9ebb9610b2f5","execution":{"iopub.status.busy":"2023-10-19T08:49:25.650279Z","iopub.execute_input":"2023-10-19T08:49:25.650624Z","iopub.status.idle":"2023-10-19T08:49:26.917617Z","shell.execute_reply.started":"2023-10-19T08:49:25.650597Z","shell.execute_reply":"2023-10-19T08:49:26.916444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_phrase = get_phrase(train_df, file_id, sequence_id)\nvisualise2d_landmarks(sequence, f\"Phrase: {sequence_phrase}\")","metadata":{"id":"iwWi9lHQOPE8","outputId":"b46ac731-b61b-41b9-f1a2-1246d219a56e","execution":{"iopub.status.busy":"2023-10-19T08:49:26.91897Z","iopub.execute_input":"2023-10-19T08:49:26.919319Z","iopub.status.idle":"2023-10-19T08:49:50.634746Z","shell.execute_reply.started":"2023-10-19T08:49:26.91929Z","shell.execute_reply":"2023-10-19T08:49:50.631848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualise(data,file_id,sequence_id):\n  path_lm = []\n  if data is supplemental_df:\n    path_lm = 'supplemental_landmarks'\n  elif data is train_df:\n    path_lm = 'train_landmarks'\n  path_to_sign = f\"/kaggle/input/asl-fingerspelling/{path_lm}/{file_id}.parquet\"\n  sign = pd.read_parquet(path_to_sign)\n  sequence = sign[sign.index == sequence_id]\n  sequence_phrase = get_phrase(data, file_id, sequence_id)\n  visualise2d_landmarks(sequence, f\"Phrase: {sequence_phrase}\")","metadata":{"id":"3g4qUXuAtdwN","execution":{"iopub.status.busy":"2023-10-19T08:49:50.637062Z","iopub.execute_input":"2023-10-19T08:49:50.638088Z","iopub.status.idle":"2023-10-19T08:49:50.644889Z","shell.execute_reply.started":"2023-10-19T08:49:50.637987Z","shell.execute_reply":"2023-10-19T08:49:50.644088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supplemental_df[supplemental_df['file_id']==1019715464]","metadata":{"id":"SSxRM0GhYjEw","outputId":"e9a9e002-8451-43d7-cbeb-5ab8fc40e180","execution":{"iopub.status.busy":"2023-10-19T08:49:50.646409Z","iopub.execute_input":"2023-10-19T08:49:50.646988Z","iopub.status.idle":"2023-10-19T08:49:50.668381Z","shell.execute_reply.started":"2023-10-19T08:49:50.64696Z","shell.execute_reply":"2023-10-19T08:49:50.666967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualise(supplemental_df,1047404576,930743248)","metadata":{"id":"y9GuiwkEukhy","outputId":"c2dac2ae-a24d-4bb4-89be-0b0f22c53ea0","execution":{"iopub.status.busy":"2023-10-19T08:49:50.670564Z","iopub.execute_input":"2023-10-19T08:49:50.671383Z","iopub.status.idle":"2023-10-19T08:50:21.907017Z","shell.execute_reply.started":"2023-10-19T08:49:50.67134Z","shell.execute_reply":"2023-10-19T08:50:21.905449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##supplemental_landmarks","metadata":{"id":"9qbuMdWsP7ZP"}},{"cell_type":"code","source":"train_df.info()","metadata":{"id":"NmWj1ndrvctC","outputId":"327a3188-46e0-443c-a703-f9a5472305d2","execution":{"iopub.status.busy":"2023-10-19T08:50:21.913426Z","iopub.execute_input":"2023-10-19T08:50:21.915236Z","iopub.status.idle":"2023-10-19T08:50:21.950704Z","shell.execute_reply.started":"2023-10-19T08:50:21.915183Z","shell.execute_reply":"2023-10-19T08:50:21.949136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain =  train_df.copy()","metadata":{"id":"UobvdS9Fxbyc","execution":{"iopub.status.busy":"2023-10-19T08:50:21.952247Z","iopub.execute_input":"2023-10-19T08:50:21.952969Z","iopub.status.idle":"2023-10-19T08:50:21.986156Z","shell.execute_reply.started":"2023-10-19T08:50:21.952921Z","shell.execute_reply":"2023-10-19T08:50:21.984197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\nimport tensorflow as tf\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit\nfrom pathlib import Path\n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport time\nimport json\nimport re\n","metadata":{"id":"APx1vfY2wg4T","execution":{"iopub.status.busy":"2023-10-19T08:50:21.987796Z","iopub.execute_input":"2023-10-19T08:50:21.98882Z","iopub.status.idle":"2023-10-19T08:50:21.997213Z","shell.execute_reply.started":"2023-10-19T08:50:21.988779Z","shell.execute_reply":"2023-10-19T08:50:21.995581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train =  train_df.copy()","metadata":{"id":"brYi3JfH06wF","execution":{"iopub.status.busy":"2023-10-19T08:50:21.99929Z","iopub.execute_input":"2023-10-19T08:50:22.000041Z","iopub.status.idle":"2023-10-19T08:50:22.027232Z","shell.execute_reply.started":"2023-10-19T08:50:21.999982Z","shell.execute_reply":"2023-10-19T08:50:22.026114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nAttempt to retrieve phrase type\nCould be used for pretraining or type specific inference\n *) Phone Number\\\n *) URL\n *3) Addres\n\"\"\"\ndef get_phrase_type(phrase):\n    # Phone Number\n    if re.match(r'^[\\d+-]+$', phrase):\n        return 'phone_number'\n    # url\n    elif any([substr in phrase for substr in ['www', '.', '/']]) and ' ' not in phrase:\n        return 'url'\n    # Address\n    else:\n        return 'address'\n\ntrain['phrase_type'] = train['phrase'].apply(get_phrase_type)","metadata":{"id":"Kl95k7jlvgsA","execution":{"iopub.status.busy":"2023-10-19T08:50:22.028663Z","iopub.execute_input":"2023-10-19T08:50:22.029837Z","iopub.status.idle":"2023-10-19T08:50:22.168901Z","shell.execute_reply.started":"2023-10-19T08:50:22.02979Z","shell.execute_reply":"2023-10-19T08:50:22.16794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"id":"zA2BWpAx42wK","outputId":"ee09ecd7-ae26-4808-8135-f81eca0c4ad4","execution":{"iopub.status.busy":"2023-10-19T08:50:22.170709Z","iopub.execute_input":"2023-10-19T08:50:22.171515Z","iopub.status.idle":"2023-10-19T08:50:22.179646Z","shell.execute_reply.started":"2023-10-19T08:50:22.171476Z","shell.execute_reply":"2023-10-19T08:50:22.178364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquet_folder = '/kaggle/input/asl-fingerspelling/train_landmarks/'\nname_parquet = [int(os.path.splitext(file)[0]) for file in os.listdir(parquet_folder) if file.endswith('.parquet')]\nprint(len(name_parquet))\ntrain = train[train['file_id'].isin(name_parquet) ]\ntrain\n","metadata":{"id":"icKtAhtv4oby","outputId":"4b221f63-9563-4fec-de4f-15fe38535469","execution":{"iopub.status.busy":"2023-10-19T08:50:22.181649Z","iopub.execute_input":"2023-10-19T08:50:22.182483Z","iopub.status.idle":"2023-10-19T08:50:22.230104Z","shell.execute_reply.started":"2023-10-19T08:50:22.182435Z","shell.execute_reply":"2023-10-19T08:50:22.228582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-fingerspelling/{path}'\n\ntrain['file_path'] = train['path'].apply(get_file_path)","metadata":{"id":"1sOxW7ZUwTko","execution":{"iopub.status.busy":"2023-10-19T08:50:22.231901Z","iopub.execute_input":"2023-10-19T08:50:22.232327Z","iopub.status.idle":"2023-10-19T08:50:22.259871Z","shell.execute_reply.started":"2023-10-19T08:50:22.232297Z","shell.execute_reply":"2023-10-19T08:50:22.258698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split Phrase To Char Tuple\ntrain['phrase_char'] = train['phrase'].apply(tuple)\n# Character Length of Phrase\ntrain['phrase_char_len'] = train['phrase_char'].apply(len)\n# Maximum Input Length\nMAX_PHRASE_LENGTH = train['phrase_char_len'].max()\nprint(f'MAX_PHRASE_LENGTH: {MAX_PHRASE_LENGTH}')\n\n# Train DataFrame indexed by sequence_id to convenientlyy lookup recording data\ntrain_sequence_id = train.set_index('sequence_id')","metadata":{"id":"HJJKA6Os0ZN3","outputId":"39ab0b27-37ce-4649-f898-f07c87e6e3a7","execution":{"iopub.status.busy":"2023-10-19T08:50:22.26227Z","iopub.execute_input":"2023-10-19T08:50:22.262771Z","iopub.status.idle":"2023-10-19T08:50:22.375457Z","shell.execute_reply.started":"2023-10-19T08:50:22.262725Z","shell.execute_reply":"2023-10-19T08:50:22.374105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"LZb-LP8ux9Km"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"id":"2kh2g9VPwmBi","outputId":"9c26b066-ce6a-4e8a-ae45-3a2132615f56","execution":{"iopub.status.busy":"2023-10-19T08:50:22.376759Z","iopub.execute_input":"2023-10-19T08:50:22.377159Z","iopub.status.idle":"2023-10-19T08:50:22.39376Z","shell.execute_reply.started":"2023-10-19T08:50:22.377127Z","shell.execute_reply":"2023-10-19T08:50:22.392605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PERCENTILES = [0.01, 0.10, 0.05, 0.25, 0.50, 0.75, 0.90, 0.95, 0.99, 0.999]\ndisplay(train['phrase_char_len'].describe(percentiles=PERCENTILES).to_frame().round(1))","metadata":{"id":"Kg8N6cyI1FYr","outputId":"f887983b-7e07-4564-ecac-b441fbdce0d9","execution":{"iopub.status.busy":"2023-10-19T08:50:22.39558Z","iopub.execute_input":"2023-10-19T08:50:22.396312Z","iopub.status.idle":"2023-10-19T08:50:22.428639Z","shell.execute_reply.started":"2023-10-19T08:50:22.396269Z","shell.execute_reply":"2023-10-19T08:50:22.427329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Character Count Occurance\nplt.figure(figsize=(15,8))\nplt.title('Character Length Occurance of Phrases')\ntrain['phrase_char_len'].value_counts().sort_index().plot(kind='bar')\nplt.xlim(-0.50, train['phrase_char_len'].max() - 1.50)\nplt.xlabel('Pharse Character Length')\nplt.ylabel('Sample Count')\nplt.grid(axis='y')\nplt.show()","metadata":{"id":"HKBQYPQI1uj_","outputId":"a58ef216-e068-4c73-8d50-c2aa32407cc9","execution":{"iopub.status.busy":"2023-10-19T08:50:22.43051Z","iopub.execute_input":"2023-10-19T08:50:22.430975Z","iopub.status.idle":"2023-10-19T08:50:23.248663Z","shell.execute_reply.started":"2023-10-19T08:50:22.430932Z","shell.execute_reply":"2023-10-19T08:50:23.247329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###Find Unique Character","metadata":{"id":"GiNpDC3u2yd-"}},{"cell_type":"code","source":"# Use Set to keep track of unique characters in phrases\nUNIQUE_CHARACTERS = set()\n\nfor phrase in tqdm(train['phrase_char']):\n    for c in phrase:\n        UNIQUE_CHARACTERS.add(c)\n# Sorted Unique Character\nUNIQUE_CHARACTERS = np.array(sorted(UNIQUE_CHARACTERS))\n# Number of Unique Characters\nN_UNIQUE_CHARACTERS = len(UNIQUE_CHARACTERS)\nprint(f'N_UNIQUE_CHARACTERS: {N_UNIQUE_CHARACTERS}')","metadata":{"id":"KTAzKEr42JVk","outputId":"388c5b91-8108-4c32-95b0-58986f9c9430","execution":{"iopub.status.busy":"2023-10-19T08:50:23.250804Z","iopub.execute_input":"2023-10-19T08:50:23.251685Z","iopub.status.idle":"2023-10-19T08:50:23.446153Z","shell.execute_reply.started":"2023-10-19T08:50:23.251639Z","shell.execute_reply":"2023-10-19T08:50:23.445049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###Example Parquet File","metadata":{"id":"QLpU7uFW241m"}},{"cell_type":"code","source":" # Read First Parquet File\nexample_parquet_df = pd.read_parquet(train['file_path'][999])\nexample_parquet_df","metadata":{"id":"ibx68QKo2a_R","outputId":"3f024af2-3b1c-48fc-8c43-9b4cb070ae47","execution":{"iopub.status.busy":"2023-10-19T08:50:23.447981Z","iopub.execute_input":"2023-10-19T08:50:23.448744Z","iopub.status.idle":"2023-10-19T08:50:28.422987Z","shell.execute_reply.started":"2023-10-19T08:50:23.448703Z","shell.execute_reply":"2023-10-19T08:50:28.421612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Each parquet file contains 1000 recordings\nprint(f'# Unique Recording: {example_parquet_df.index.nunique()}')","metadata":{"id":"FJMvcpNo3gZq","outputId":"f49559e9-1cbb-43d5-ad7a-e5a0e5a277f9","execution":{"iopub.status.busy":"2023-10-19T08:50:28.424504Z","iopub.execute_input":"2023-10-19T08:50:28.42494Z","iopub.status.idle":"2023-10-19T08:50:28.434586Z","shell.execute_reply.started":"2023-10-19T08:50:28.424902Z","shell.execute_reply":"2023-10-19T08:50:28.433727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"id":"BuRFrhwQ2_Aq","outputId":"689561b3-0d06-43db-e3e0-0298d1e119e9","execution":{"iopub.status.busy":"2023-10-19T08:50:28.435893Z","iopub.execute_input":"2023-10-19T08:50:28.436702Z","iopub.status.idle":"2023-10-19T08:50:28.471188Z","shell.execute_reply.started":"2023-10-19T08:50:28.436657Z","shell.execute_reply":"2023-10-19T08:50:28.469771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of parquet chunks to analyse\nN = 1\n# Number of Unique Frames in Recording\nN_UNIQUE_FRAMES = []\nSEED = 42\nUNIQUE_FILE_PATHS = pd.Series(train['file_path'].unique())\nUNIQUE_FILE_PATHS\nfor idx, file_path in enumerate(tqdm(UNIQUE_FILE_PATHS.sample(N, random_state=SEED))):\n    df = pd.read_parquet(file_path)\n    for group, group_df in df.groupby('sequence_id'):\n        N_UNIQUE_FRAMES.append(group_df['frame'].nunique())\n\n# Convert to Numpy Array\nN_UNIQUE_FRAMES = np.array(N_UNIQUE_FRAMES)","metadata":{"id":"rlIGh7m33EwI","outputId":"8ed3760d-b99e-4a4a-8f2a-3211b91148e0","execution":{"iopub.status.busy":"2023-10-19T08:50:28.472653Z","iopub.execute_input":"2023-10-19T08:50:28.472968Z","iopub.status.idle":"2023-10-19T08:50:33.155681Z","shell.execute_reply.started":"2023-10-19T08:50:28.472942Z","shell.execute_reply":"2023-10-19T08:50:33.154587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_UNIQUE_FRAMES","metadata":{"id":"2h9meoe2NSeV","outputId":"3d241cbf-970e-4a8f-ac42-954130d7f2f5","execution":{"iopub.status.busy":"2023-10-19T08:50:33.156987Z","iopub.execute_input":"2023-10-19T08:50:33.157781Z","iopub.status.idle":"2023-10-19T08:50:33.165325Z","shell.execute_reply.started":"2023-10-19T08:50:33.157748Z","shell.execute_reply":"2023-10-19T08:50:33.16436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(pd.Series(N_UNIQUE_FRAMES).describe(percentiles=PERCENTILES).to_frame('Value').astype(int))","metadata":{"id":"PuAXJYAeP-h0","outputId":"7e6cb445-c5a8-419b-8a51-1d425173d2be","execution":{"iopub.status.busy":"2023-10-19T08:50:33.166694Z","iopub.execute_input":"2023-10-19T08:50:33.167045Z","iopub.status.idle":"2023-10-19T08:50:33.193161Z","shell.execute_reply.started":"2023-10-19T08:50:33.167017Z","shell.execute_reply":"2023-10-19T08:50:33.191732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.title('Number of Unique Frames', size = 24)\npd.Series(N_UNIQUE_FRAMES).plot(kind='hist',bins=150)\nxlim = math.ceil(plt.xlim()[1])\nplt.xlim(0, xlim)\n","metadata":{"id":"PeQ7pk0xQC3N","outputId":"d66b81d1-b703-4f8b-ab89-a3bf1c00e6b3","execution":{"iopub.status.busy":"2023-10-19T08:50:33.194681Z","iopub.execute_input":"2023-10-19T08:50:33.195298Z","iopub.status.idle":"2023-10-19T08:50:33.74316Z","shell.execute_reply.started":"2023-10-19T08:50:33.195265Z","shell.execute_reply":"2023-10-19T08:50:33.742008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# With N_TARGET_FRAMES = 256 ~85% will be below\nN_UNIQUE_FRAMES_WATERFALL = []\n# Maximum Number of Unique Frames to use\nN_MAX_UNIQUE_FRAMES = 400\n# Compute Percentage\nfor n in tqdm(range(0,N_MAX_UNIQUE_FRAMES+1)):\n    N_UNIQUE_FRAMES_WATERFALL.append(sum(N_UNIQUE_FRAMES >= n) / len(N_UNIQUE_FRAMES) * 100)\n\nplt.figure(figsize=(18,10))\nplt.title('Waterfall Plot For Number Of Unique Frames')\npd.Series(N_UNIQUE_FRAMES_WATERFALL).plot(kind='bar')\nplt.grid(axis='y')\nplt.xticks([1] + np.arange(5, N_MAX_UNIQUE_FRAMES+5, 5).tolist(), size=8, rotation=45)\nplt.xlabel('Number of Unique Frames', size=16)\nplt.yticks(np.arange(0, 100+5, 5), [f'{i}%' for i in range(0,100+5,5)])\nplt.ylim(0, 100)\nplt.ylabel('Percentage of Samples With At Least N Unique Frames', size=16)\nplt.show()","metadata":{"id":"rtHibUgBQNtD","outputId":"bcc3a336-9eb5-4681-9f9b-b5968e9f84a4","execution":{"iopub.status.busy":"2023-10-19T08:50:33.744976Z","iopub.execute_input":"2023-10-19T08:50:33.74569Z","iopub.status.idle":"2023-10-19T08:50:35.380498Z","shell.execute_reply.started":"2023-10-19T08:50:33.745639Z","shell.execute_reply":"2023-10-19T08:50:35.379196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Landmark Indices","metadata":{"id":"efNoxYBWSQpc"}},{"cell_type":"code","source":"def get_idxs(df, words_pos, words_neg=[], ret_names=True, idxs_pos=None):\n    idxs = []\n    names = []\n    for w in words_pos:\n        for col_idx, col in enumerate(example_parquet_df.columns):\n            # Exclude Non Landmark Columns\n            if col in ['frame']:\n                continue\n\n            col_idx = int(col.split('_')[-1])\n            # Check if column name contains all words\n            if (w in col) and (idxs_pos is None or col_idx in idxs_pos) and all([w not in col for w in words_neg]):\n                idxs.append(col_idx)\n                names.append(col)\n    # Convert to Numpy arrays\n    idxs = np.array(idxs)\n    names = np.array(names)\n    # Returns either both column indices and names\n    if ret_names:\n        return idxs, names\n    # Or only columns indices\n    else:\n        return idxs","metadata":{"id":"Xd5GOhO1SLNH","execution":{"iopub.status.busy":"2023-10-19T08:50:35.382611Z","iopub.execute_input":"2023-10-19T08:50:35.383115Z","iopub.status.idle":"2023-10-19T08:50:35.392247Z","shell.execute_reply.started":"2023-10-19T08:50:35.383056Z","shell.execute_reply":"2023-10-19T08:50:35.391016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lips Landmark Face Ids\nLIPS_LANDMARK_IDXS = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\n\n# Landmark Indices for Left/Right hand without z axis in raw data\nLEFT_HAND_IDXS0, LEFT_HAND_NAMES0 = get_idxs(example_parquet_df, ['left_hand'], ['z'])\nRIGHT_HAND_IDXS0, RIGHT_HAND_NAMES0 = get_idxs(example_parquet_df, ['right_hand'], ['z'])\nLIPS_IDXS0, LIPS_NAMES0 = get_idxs(example_parquet_df, ['face'], ['z'], idxs_pos=LIPS_LANDMARK_IDXS)\nCOLUMNS0 = np.concatenate((LEFT_HAND_NAMES0, RIGHT_HAND_NAMES0, LIPS_NAMES0))\nN_COLS0 = len(COLUMNS0)\n# Only X/Y axes are used\nN_DIMS0 = 2\nprint(f'N_COLS0: {N_COLS0}')","metadata":{"id":"phgFz1_hSTK0","outputId":"ba651a1c-0ba5-40b2-b452-2d6cbf7509c8","execution":{"iopub.status.busy":"2023-10-19T08:50:35.393795Z","iopub.execute_input":"2023-10-19T08:50:35.394201Z","iopub.status.idle":"2023-10-19T08:50:35.414042Z","shell.execute_reply.started":"2023-10-19T08:50:35.394172Z","shell.execute_reply":"2023-10-19T08:50:35.412776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Landmark Indices in subset of dataframe with only COLUMNS selected\nLEFT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, LEFT_HAND_NAMES0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, RIGHT_HAND_NAMES0)).squeeze()\nLIPS_IDXS = np.argwhere(np.isin(COLUMNS0, LIPS_NAMES0)).squeeze()\nN_COLS = N_COLS0\n# Only X/Y axes are used\nN_DIMS = 2\nprint(f'N_COLS: {N_COLS}')\nRIGHT_HAND_IDXS","metadata":{"id":"VZHIrGJ5ULWx","outputId":"9f9eef0f-20c4-4786-fb12-ab94d61001e8","execution":{"iopub.status.busy":"2023-10-19T08:50:35.415601Z","iopub.execute_input":"2023-10-19T08:50:35.416559Z","iopub.status.idle":"2023-10-19T08:50:35.435353Z","shell.execute_reply.started":"2023-10-19T08:50:35.416517Z","shell.execute_reply":"2023-10-19T08:50:35.434147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Indices in processed data by axes with only dominant hand\nHAND_X_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'x' in name]\n    ).squeeze()\nHAND_Y_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'y' in name]\n    ).squeeze()\n# Names in processed data by axes\nHAND_X_NAMES = LEFT_HAND_NAMES0[HAND_X_IDXS]\nHAND_Y_NAMES = LEFT_HAND_NAMES0[HAND_Y_IDXS]\nHAND_Y_IDXS","metadata":{"id":"H24liUBfU67u","outputId":"ba7822b8-405a-4f24-eecf-d6d69f6fe322","execution":{"iopub.status.busy":"2023-10-19T08:50:35.437114Z","iopub.execute_input":"2023-10-19T08:50:35.438389Z","iopub.status.idle":"2023-10-19T08:50:35.447462Z","shell.execute_reply.started":"2023-10-19T08:50:35.438356Z","shell.execute_reply":"2023-10-19T08:50:35.44643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEFT_HAND_NAMES0","metadata":{"id":"5P4_SdDYQHdi","outputId":"b11a6742-5e26-416b-a022-f6a70f59bd6b","execution":{"iopub.status.busy":"2023-10-19T08:50:35.453207Z","iopub.execute_input":"2023-10-19T08:50:35.454291Z","iopub.status.idle":"2023-10-19T08:50:35.460285Z","shell.execute_reply.started":"2023-10-19T08:50:35.454257Z","shell.execute_reply":"2023-10-19T08:50:35.459256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HAND_X_NAMES","metadata":{"id":"EqkEL_0oVKa-","outputId":"bca5e707-d88f-448c-8e63-05d848a3c5c1","execution":{"iopub.status.busy":"2023-10-19T08:50:35.461592Z","iopub.execute_input":"2023-10-19T08:50:35.462542Z","iopub.status.idle":"2023-10-19T08:50:35.475344Z","shell.execute_reply.started":"2023-10-19T08:50:35.462507Z","shell.execute_reply":"2023-10-19T08:50:35.474167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Number Of Non-NaN Frames","metadata":{"id":"RcXhG1hIWB1I"}},{"cell_type":"code","source":"\"\"\"\n    Tensorflow layer to process data in TFLite\n    Data needs to be processed in the model itself, so we can not use Python\n\"\"\"\nclass PreprocessLayerNonNaN(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayerNonNaN, self).__init__()\n\n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,N_COLS0], dtype=tf.float32),),\n    )\n    def call(self, data0):\n        # Fill NaN Values With 0\n        data = tf.where(tf.math.is_nan(data0), 0.0, data0)\n\n        # Hacky\n        data = data[None]\n\n        # Empty Hand Frame Filtering\n        hands = tf.slice(data, [0,0,0], [-1, -1, 84])\n        hands = tf.abs(hands)\n        mask = tf.reduce_sum(hands, axis=2)\n        mask = tf.not_equal(mask, 0)\n        data = data[mask][None]\n        data = tf.squeeze(data, axis=[0])\n\n        return data\n\npreprocess_layer_non_nan = PreprocessLayerNonNaN()","metadata":{"id":"Popg97CwVIfZ","execution":{"iopub.status.busy":"2023-10-19T08:50:35.477347Z","iopub.execute_input":"2023-10-19T08:50:35.478097Z","iopub.status.idle":"2023-10-19T08:50:35.495232Z","shell.execute_reply.started":"2023-10-19T08:50:35.478052Z","shell.execute_reply":"2023-10-19T08:50:35.494273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['file_path'][0]","metadata":{"id":"Hlc8T_3_dP9e","outputId":"c42afeec-b798-4cbd-f2b8-54a93a88e1bb","execution":{"iopub.status.busy":"2023-10-19T08:50:35.49731Z","iopub.execute_input":"2023-10-19T08:50:35.497683Z","iopub.status.idle":"2023-10-19T08:50:35.514692Z","shell.execute_reply.started":"2023-10-19T08:50:35.497651Z","shell.execute_reply":"2023-10-19T08:50:35.513286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_parquet('/kaggle/input/asl-fingerspelling/train_landmarks/5414471.parquet')\ndf = df.head(5)\ndf","metadata":{"id":"ayC3E2CYcw3S","outputId":"2d2e3b28-00a9-4599-b1f1-42c34cc56459","execution":{"iopub.status.busy":"2023-10-19T08:50:35.51597Z","iopub.execute_input":"2023-10-19T08:50:35.516413Z","iopub.status.idle":"2023-10-19T08:50:39.66139Z","shell.execute_reply.started":"2023-10-19T08:50:35.516372Z","shell.execute_reply":"2023-10-19T08:50:39.660123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unique Parquet Files\nUNIQUE_FILE_PATHS = pd.Series(train['file_path'].unique())\nUNIQUE_FILE_PATHS\n# Number of parquet chunks to analyse\nN = 1\n# Number of Non Nan Frames in Recording\nN_NON_NAN_FRAMES = []\n\nfor idx, file_path in enumerate(tqdm(UNIQUE_FILE_PATHS.sample(N, random_state=SEED))):\n    df = pd.read_parquet(file_path)\n    for group, group_df in df.groupby('sequence_id'):\n        frames = preprocess_layer_non_nan(group_df[COLUMNS0].values).numpy()\n        N_NON_NAN_FRAMES.append(len(frames))","metadata":{"id":"LdgGtb8vWEjQ","outputId":"e1a2a506-2c78-41a3-b267-b8a4539dd410","execution":{"iopub.status.busy":"2023-10-19T08:50:39.662686Z","iopub.execute_input":"2023-10-19T08:50:39.663029Z","iopub.status.idle":"2023-10-19T08:50:46.409481Z","shell.execute_reply.started":"2023-10-19T08:50:39.662986Z","shell.execute_reply":"2023-10-19T08:50:46.408312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert to Numpy Array\nN_NON_NAN_FRAMES = pd.Series(N_NON_NAN_FRAMES).to_frame('# Frames')","metadata":{"id":"ZRXIHwG8WNGs","execution":{"iopub.status.busy":"2023-10-19T08:50:46.411541Z","iopub.execute_input":"2023-10-19T08:50:46.411973Z","iopub.status.idle":"2023-10-19T08:50:46.41875Z","shell.execute_reply.started":"2023-10-19T08:50:46.411933Z","shell.execute_reply":"2023-10-19T08:50:46.417496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(N_NON_NAN_FRAMES.describe(percentiles=PERCENTILES))","metadata":{"id":"ugQatzOZXYYe","outputId":"c8bf148c-fe3f-4745-9d32-41ae6fe8df88","execution":{"iopub.status.busy":"2023-10-19T08:50:46.420543Z","iopub.execute_input":"2023-10-19T08:50:46.421062Z","iopub.status.idle":"2023-10-19T08:50:46.444553Z","shell.execute_reply.started":"2023-10-19T08:50:46.42102Z","shell.execute_reply":"2023-10-19T08:50:46.442975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_NON_NAN_FRAMES.plot(kind='hist', bins=128, figsize=(15,8))\nplt.title('Number of Non NaN Frames', size=24)\nxlim = math.ceil(plt.xlim()[1])\nplt.xlim(0, xlim)\n","metadata":{"id":"0pkaCeNpXjEF","outputId":"6f5670f8-0656-4f78-d11b-73ecf0a065b4","execution":{"iopub.status.busy":"2023-10-19T08:50:46.446181Z","iopub.execute_input":"2023-10-19T08:50:46.446551Z","iopub.status.idle":"2023-10-19T08:50:46.940714Z","shell.execute_reply.started":"2023-10-19T08:50:46.446522Z","shell.execute_reply":"2023-10-19T08:50:46.939516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###Tensorflow Preprocess Layer","metadata":{"id":"XSoz2Uu7Yv8H"}},{"cell_type":"code","source":"\"\"\"\n    Tensorflow layer to process data in TFLite\n    Data needs to be processed in the model itself, so we can not use Python\n\"\"\"\nN_TARGET_FRAMES = 128\nclass PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n\n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,N_COLS0], dtype=tf.float32),),\n    )\n    def call(self, data0, resize=True):\n        # Fill NaN Values With 0\n        data = tf.where(tf.math.is_nan(data0), 0.0, data0)\n\n        # Hacky\n        data = data[None]\n\n        # Empty Hand Frame Filtering\n        hands = tf.slice(data, [0,0,0], [-1, -1, 84])\n        hands = tf.abs(hands)\n        mask = tf.reduce_sum(hands, axis=2)\n        mask = tf.not_equal(mask, 0)\n        data = data[mask][None]\n\n        # Pad Zeros\n        N_FRAMES = len(data[0])\n        if N_FRAMES < N_TARGET_FRAMES:\n            data = tf.concat((\n                data,\n                tf.zeros([1,N_TARGET_FRAMES-N_FRAMES,N_COLS], dtype=tf.float32)\n            ), axis=1)\n        # Downsample\n        data = tf.image.resize(\n            data,\n            [1, N_TARGET_FRAMES],\n            method=tf.image.ResizeMethod.BILINEAR,\n        )\n\n        # Squeeze Batch Dimension\n        data = tf.squeeze(data, axis=[0])\n\n        return data\n\npreprocess_layer = PreprocessLayer()\n\ninputs = group_df[COLUMNS0].values\ninputs = inputs[:1]\n\nframes = preprocess_layer(inputs)\n\nprint(f'inputs shape: {inputs.shape}')\nprint(f'frames shape: {frames.shape}, NaN count: {np.isnan(frames).sum()}')","metadata":{"id":"JQWwLU-6YFmB","outputId":"aad84fa7-dc5c-410d-eb1e-20de720b11be","execution":{"iopub.status.busy":"2023-10-19T08:50:46.942455Z","iopub.execute_input":"2023-10-19T08:50:46.942926Z","iopub.status.idle":"2023-10-19T08:50:47.065363Z","shell.execute_reply.started":"2023-10-19T08:50:46.942884Z","shell.execute_reply":"2023-10-19T08:50:47.063812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.shape","metadata":{"id":"kuCy5lYpRdEJ","outputId":"0661216b-a97e-47ff-8022-7d02542b753a","execution":{"iopub.status.busy":"2023-10-19T08:50:47.067123Z","iopub.execute_input":"2023-10-19T08:50:47.067575Z","iopub.status.idle":"2023-10-19T08:50:47.075207Z","shell.execute_reply.started":"2023-10-19T08:50:47.067529Z","shell.execute_reply":"2023-10-19T08:50:47.073983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Target Arrays Processed Input Videos\nN_SAMPLES = len(train)\nX = np.zeros([N_SAMPLES, N_TARGET_FRAMES, N_COLS], dtype=np.float32)\n# Ordinally Encoded Target With value 59 for pad token\ny = np.full(shape=[N_SAMPLES, N_TARGET_FRAMES], fill_value=N_UNIQUE_CHARACTERS, dtype=np.int8)\n# Phrase Type\ny_phrase_type = np.empty(shape=[N_SAMPLES], dtype=object)\nX.shape  #(num_data, N_TARGET_FRAMES , feature )","metadata":{"id":"0GMzqiktYlUm","outputId":"8cc546dc-258b-4961-f477-513632f4b2c0","execution":{"iopub.status.busy":"2023-10-19T08:50:47.076624Z","iopub.execute_input":"2023-10-19T08:50:47.077075Z","iopub.status.idle":"2023-10-19T08:50:47.859601Z","shell.execute_reply.started":"2023-10-19T08:50:47.077029Z","shell.execute_reply":"2023-10-19T08:50:47.858045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"id":"O0szrj6fkfJY","outputId":"f1913823-33ec-4625-ac33-ae6300fb4ae4","execution":{"iopub.status.busy":"2023-10-19T08:50:47.86123Z","iopub.execute_input":"2023-10-19T08:50:47.861899Z","iopub.status.idle":"2023-10-19T08:50:47.871925Z","shell.execute_reply.started":"2023-10-19T08:50:47.861861Z","shell.execute_reply":"2023-10-19T08:50:47.870255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read Character to Ordinal Encoding Mapping\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    CHAR2ORD = json.load(json_file)\n\n# Character to Ordinal Encoding Mapping\ndisplay(pd.Series(CHAR2ORD).to_frame('Ordinal Encoding'))\nN_UNIQUE_CHARACTERSPAD_TOKEN = len(CHAR2ORD)\nSOS_TOKEN = len(CHAR2ORD) + 1 # Start Of Sentence\nEOS_TOKEN = len(CHAR2ORD) + 2 # End Of Sentence","metadata":{"id":"Lxe5DfSbkY1q","outputId":"0b0e8879-0ab1-48f8-bd06-175cd6551c38","execution":{"iopub.status.busy":"2023-10-19T08:50:47.873974Z","iopub.execute_input":"2023-10-19T08:50:47.875108Z","iopub.status.idle":"2023-10-19T08:50:47.903108Z","shell.execute_reply.started":"2023-10-19T08:50:47.874986Z","shell.execute_reply":"2023-10-19T08:50:47.901764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# All Unique Parquet Files\nparquet_folder = '/kaggle/input/asl-fingerspelling/train_landmarks/'\nname_parquet = [int(os.path.splitext(file)[0]) for file in os.listdir(parquet_folder) if file.endswith('.parquet')]\nprint(len(name_parquet))\ntrain = train[train['file_id'].isin(name_parquet) ]\ntrain\n# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-fingerspelling/{path}'\ntrain['file_path'] = train['path'].apply(get_file_path)\nUNIQUE_FILE_PATHS = pd.Series(train['file_path'].unique())\nN_UNIQUE_FILE_PATHS = len(UNIQUE_FILE_PATHS)\nUNIQUE_FILE_PATHS","metadata":{"id":"BmkXnGO7oQlC","outputId":"212c90a9-bb8c-460a-9b64-3fb9e7add2ef","execution":{"iopub.status.busy":"2023-10-19T08:50:47.905047Z","iopub.execute_input":"2023-10-19T08:50:47.905936Z","iopub.status.idle":"2023-10-19T08:50:47.97153Z","shell.execute_reply.started":"2023-10-19T08:50:47.90589Z","shell.execute_reply":"2023-10-19T08:50:47.969609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Counter to keep track of sample\nrow = 0\ncount = 0\n# Compressed Parquet Files\nPath('train_landmark_subsets').mkdir(parents=True, exist_ok=True)\n# Numbre Of Frames Per Character\nN_FRAMES_PER_CHARACTER = []\n# Minimum Number Of Frames Per Character\nMIN_NUM_FRAMES_PER_CHARACTER = 4\nVALID_IDXS = []\n\n# Fill Arrays\nfor idx, file_path in enumerate(tqdm(UNIQUE_FILE_PATHS)):\n    # Progress Logging\n    print(f'Processed {idx:02d}/{N_UNIQUE_FILE_PATHS} parquet files')\n    # Read parquet file\n    df = pd.read_parquet(file_path)\n    # Save COLUMN Subset of parquet files for TFLite Model verficiation\n    name = file_path.split('/')[-1]\n    if idx < 10:\n        df[COLUMNS0].to_parquet(f'train_landmark_subsets/{name}', engine='pyarrow', compression='zstd')\n    # Iterate Over Samples\n    for group, group_df in df.groupby('sequence_id'):\n        # Number of Frames Per Character\n        n_frames_per_character =  len(group_df[COLUMNS0].values) / len(train_sequence_id.loc[group, 'phrase_char'])\n        N_FRAMES_PER_CHARACTER.append(n_frames_per_character)\n        if n_frames_per_character < MIN_NUM_FRAMES_PER_CHARACTER:\n            count = count + 1\n            continue\n        else:\n            # Add Valid Index\n            VALID_IDXS.append(count)\n            count = count + 1\n\n        # Get Processed Frames and non empty frame indices\n        frames = preprocess_layer(group_df[COLUMNS0].values)\n        assert frames.ndim == 2\n        # Assign\n        X[row] = frames\n        # Add Target By Ordinally Encoding Characters\n        phrase_char = train_sequence_id.loc[group, 'phrase_char']\n        for col, char in enumerate(phrase_char):\n            y[row, col] = CHAR2ORD.get(char)\n        # Add EOS Token\n        y[row, col+1] = EOS_TOKEN\n        # Phrase Type\n        y_phrase_type[row] = train_sequence_id.loc[group, 'phrase_type']\n        # Row Count\n        row += 1\n    # clean up\n    gc.collect()","metadata":{"id":"wWOs_e1wjwgk","outputId":"565f51d9-5600-4c7f-95b9-40d502e48cbd","execution":{"iopub.status.busy":"2023-10-19T08:50:47.973186Z","iopub.execute_input":"2023-10-19T08:50:47.973549Z","iopub.status.idle":"2023-10-19T09:10:29.672102Z","shell.execute_reply.started":"2023-10-19T08:50:47.973518Z","shell.execute_reply":"2023-10-19T09:10:29.671123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"id":"qeQcdNmDqpCJ","outputId":"55629375-92a1-4919-ef68-1329e5cdd64a","execution":{"iopub.status.busy":"2023-10-19T09:10:29.675431Z","iopub.execute_input":"2023-10-19T09:10:29.676221Z","iopub.status.idle":"2023-10-19T09:10:29.684171Z","shell.execute_reply.started":"2023-10-19T09:10:29.676175Z","shell.execute_reply":"2023-10-19T09:10:29.682892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_reduce = pd.read_parquet('/kaggle/working/train_landmark_subsets/5414471.parquet')\ndata_bef_reduce = pd.read_parquet('/kaggle/input/asl-fingerspelling/train_landmarks/5414471.parquet')","metadata":{"id":"XhgJ0IoyqXz5","execution":{"iopub.status.busy":"2023-10-19T09:12:06.41218Z","iopub.execute_input":"2023-10-19T09:12:06.412757Z","iopub.status.idle":"2023-10-19T09:12:12.561609Z","shell.execute_reply.started":"2023-10-19T09:12:06.412716Z","shell.execute_reply":"2023-10-19T09:12:12.560515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_bef_reduce","metadata":{"id":"rqarkAkeu8od","outputId":"e25a46db-602b-4b7d-a8a3-679ae3af0d9f","execution":{"iopub.status.busy":"2023-10-19T09:12:18.734641Z","iopub.execute_input":"2023-10-19T09:12:18.735114Z","iopub.status.idle":"2023-10-19T09:12:20.008706Z","shell.execute_reply.started":"2023-10-19T09:12:18.735082Z","shell.execute_reply":"2023-10-19T09:12:20.00741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rows denotes the number of samples with frames/character above threshold\nprint(f'row: {row}, count: {count}')","metadata":{"id":"oEryUXVhkSFS","outputId":"94f7f375-edc2-479a-9528-678950304d4d","execution":{"iopub.status.busy":"2023-10-19T09:12:26.354512Z","iopub.execute_input":"2023-10-19T09:12:26.355479Z","iopub.status.idle":"2023-10-19T09:12:26.3615Z","shell.execute_reply.started":"2023-10-19T09:12:26.355444Z","shell.execute_reply":"2023-10-19T09:12:26.360361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example target, note the phrase is padded with the pad token 59\nprint(f'Example Target: {y[0]}')","metadata":{"id":"kId9aAQnmaxz","outputId":"95404fed-9836-41fa-92f7-b31f8767c950","execution":{"iopub.status.busy":"2023-10-19T09:12:30.207509Z","iopub.execute_input":"2023-10-19T09:12:30.207917Z","iopub.status.idle":"2023-10-19T09:12:30.21444Z","shell.execute_reply.started":"2023-10-19T09:12:30.207885Z","shell.execute_reply":"2023-10-19T09:12:30.213279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filer X/y\nX = X[:row]\ny = y[:row]\ny.shape","metadata":{"id":"NcxVMA7smbHD","outputId":"21b0074f-8b37-4298-cb20-0c416fab7bc3","execution":{"iopub.status.busy":"2023-10-19T09:12:32.886569Z","iopub.execute_input":"2023-10-19T09:12:32.886958Z","iopub.status.idle":"2023-10-19T09:12:32.895214Z","shell.execute_reply.started":"2023-10-19T09:12:32.886929Z","shell.execute_reply":"2023-10-19T09:12:32.893691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"id":"DeTMs2NOomob","outputId":"ccd50618-8ea4-4df4-9842-9d07d1303020","execution":{"iopub.status.busy":"2023-10-19T09:12:40.214469Z","iopub.execute_input":"2023-10-19T09:12:40.214862Z","iopub.status.idle":"2023-10-19T09:12:40.222207Z","shell.execute_reply.started":"2023-10-19T09:12:40.214834Z","shell.execute_reply":"2023-10-19T09:12:40.220959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save X/y\nnp.save('X.npy', X)\nnp.save('y.npy', y)\n# Save Validation\nsplitter = GroupShuffleSplit(test_size=0.10, n_splits=2, random_state=SEED)\nPARTICIPANT_IDS = train['participant_id'].values[VALID_IDXS]\ntrain_idxs, val_idxs = next(splitter.split(X, y, groups=PARTICIPANT_IDS))\n\n# Save Train\nnp.save('X_train.npy', X[train_idxs])\nnp.save('y_train.npy', y[train_idxs])\n# Save Validation\nnp.save('X_val.npy', X[val_idxs])\nnp.save('y_val.npy', y[val_idxs])\n# Verify Train/Val is correctly split by participan id\nprint(f'Patient ID Intersection Train/Val: {set(PARTICIPANT_IDS[train_idxs]).intersection(PARTICIPANT_IDS[val_idxs])}')\n# Train/Val Sizes\nprint(f'# Train Samples: {len(train_idxs)}, # Val Samples: {len(val_idxs)}')","metadata":{"id":"fNpSyWDImdxO","outputId":"d91300b6-e416-44d8-f511-4f3cc14833c3","execution":{"iopub.status.busy":"2023-10-19T09:12:53.186947Z","iopub.execute_input":"2023-10-19T09:12:53.1882Z","iopub.status.idle":"2023-10-19T09:13:34.77369Z","shell.execute_reply.started":"2023-10-19T09:12:53.188162Z","shell.execute_reply":"2023-10-19T09:13:34.772249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_FRAMES_PER_CHARACTER_S = pd.Series(N_FRAMES_PER_CHARACTER)\n\ndisplay(N_FRAMES_PER_CHARACTER_S.describe(percentiles=PERCENTILES).to_frame('Value').round(2))\n\nplt.figure(figsize=(20,10))\nplt.title('Number Of Frames Per Phrase Character')\nN_FRAMES_PER_CHARACTER_S.plot(kind='hist', bins=128)\n# Plot till 99th percentile\np99 = math.ceil(np.percentile(N_FRAMES_PER_CHARACTER_S, 99))\nplt.xticks(np.arange(0, p99+1, 1))\nplt.xlim(0, p99)\nplt.xlabel('Number Of Frames Per Phrase Character')\nplt.ylabel('Sample Count')\nplt.grid()\nplt.show()","metadata":{"id":"Kqmsdy6QmfDd","outputId":"09a57930-999b-44a5-9b96-fc6acb44c322","execution":{"iopub.status.busy":"2023-10-19T09:29:05.374864Z","iopub.execute_input":"2023-10-19T09:29:05.375365Z","iopub.status.idle":"2023-10-19T09:29:05.903055Z","shell.execute_reply.started":"2023-10-19T09:29:05.375333Z","shell.execute_reply":"2023-10-19T09:29:05.901881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_left_right_hand_mean_std():\n    # Dominant Hand Statistics\n    MEANS = np.zeros([N_COLS], dtype=np.float32)\n    STDS = np.zeros([N_COLS], dtype=np.float32)\n\n    # Plot\n    fig, axes = plt.subplots(3, figsize=(20, 3*8))\n\n    # Iterate over all landmarks\n    for col, v in enumerate(tqdm(X.reshape([-1, N_COLS]).T)):\n        v = v[np.nonzero(v)]\n        # Remove zero values as they are NaN values\n        MEANS[col] = v.astype(np.float32).mean()\n        STDS[col] = v.astype(np.float32).std()\n        if col in LEFT_HAND_IDXS:\n            axes[0].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n        elif col in RIGHT_HAND_IDXS:\n            axes[1].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n        else:\n            axes[2].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n    for ax, name in zip(axes, ['Left Hand', 'Right Hand', 'Lips']):\n        ax.set_title(f'{name}', size=24)\n        ax.tick_params(axis='x', labelsize=8, rotation=45)\n        ax.set_ylim(0.0, 1.0)\n        ax.grid(axis='y')\n\n    plt.show()\n\n    return MEANS, STDS\n\n# Get Dominant Hand Mean/Standard Deviation\nMEANS, STDS = get_left_right_hand_mean_std()\n# Save Mean/STD to normalize input in neural network model\nnp.save('MEANS.npy', MEANS)\nnp.save('STDS.npy', STDS)","metadata":{"id":"Fw82wmFom-yE","outputId":"51142212-83bd-4891-aa01-960de64a60fb","execution":{"iopub.status.busy":"2023-10-19T09:29:20.055316Z","iopub.execute_input":"2023-10-19T09:29:20.056205Z","iopub.status.idle":"2023-10-19T09:31:04.854647Z","shell.execute_reply.started":"2023-10-19T09:29:20.056162Z","shell.execute_reply":"2023-10-19T09:31:04.853594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"tdBwKbLenCVc"},"execution_count":null,"outputs":[]}]}