{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nimport plotly.io as pio\nfrom pathlib import Path\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport json\nfrom tqdm import tqdm\n\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import KMeans\nfrom matplotlib.colors import ListedColormap\nfrom sklearn.preprocessing import normalize\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-13T07:56:09.544047Z","iopub.execute_input":"2023-06-13T07:56:09.544482Z","iopub.status.idle":"2023-06-13T07:56:11.872892Z","shell.execute_reply.started":"2023-06-13T07:56:09.54445Z","shell.execute_reply":"2023-06-13T07:56:11.871842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualise2d_landmarks(parquet_df, title=\"\",inter_frame_delay : int = 100):\n    connections = [  \n        [0, 1, 2, 3, 4,],\n        [0, 5, 6, 7, 8],\n        [0, 9, 10, 11, 12],\n        [0, 13, 14, 15, 16],\n        [0, 17, 18, 19, 20],\n\n        \n        [38, 36, 35, 34, 30, 31, 32, 33, 37],\n        [40, 39],\n        [52, 46, 50, 48, 46, 44, 42, 41, 43, 45, 47, 49, 45, 51],\n        [42, 54, 56, 58, 60, 62, 58],\n        [41, 53, 55, 57, 59, 61, 57],\n        [54, 53],\n\n        \n        [80, 81, 82, 83, 84, ],\n        [80, 85, 86, 87, 88],\n        [80, 89, 90, 91, 92],\n        [80, 93, 94, 95, 96],\n        [80, 97, 98, 99, 100], ]\n\n    parquet_df = map_new_to_old_style(parquet_df)\n    frames = sorted(set(parquet_df.frame))\n    first_frame = min(frames)\n    parquet_df['color'] = parquet_df.type.apply(lambda row: assign_color(row))\n    parquet_df['plot_order'] = parquet_df.apply(lambda row: assign_order(row), axis=1)\n    first_frame_df = parquet_df[parquet_df.frame == first_frame].copy()\n    first_frame_df = first_frame_df.sort_values([\"plot_order\"]).set_index('plot_order')\n\n\n    frames_l = []\n    for frame in frames:\n        filtered_df = parquet_df[parquet_df.frame == frame].copy()\n        filtered_df = filtered_df.sort_values([\"plot_order\"]).set_index(\"plot_order\")\n        traces = [go.Scatter(\n            x=filtered_df['x'],\n            y=filtered_df['y'],\n            mode='markers',\n            marker=dict(\n                color=filtered_df.color,\n                size=9))]\n\n        for i, seg in enumerate(connections):\n            trace = go.Scatter(\n                    x=filtered_df.loc[seg]['x'],\n                    y=filtered_df.loc[seg]['y'],\n                    mode='lines',\n            )\n            traces.append(trace)\n        frame_data = go.Frame(data=traces, traces = [i for i in range(17)])\n        frames_l.append(frame_data)\n\n    traces = [go.Scatter(\n        x=first_frame_df['x'],\n        y=first_frame_df['y'],\n        mode='markers',\n        marker=dict(\n            color=first_frame_df.color,\n            size=9\n        )\n    )]\n    for i, seg in enumerate(connections):\n        trace = go.Scatter(\n            x=first_frame_df.loc[seg]['x'],\n            y=first_frame_df.loc[seg]['y'],\n            mode='lines',\n            line=dict(\n                color='black',\n                width=2\n            )\n        )\n        traces.append(trace)\n    \n    fig = go.Figure(\n            data=traces,\n            frames=frames_l\n        )\n\n    fig.update_layout(\n        width=500,\n        height=800,\n        scene={\n            'aspectmode': 'data',\n        },\n        updatemenus=[\n            {\n                \"buttons\": [\n                    {\n                        \"args\": [None, {\"frame\": {\"duration\": inter_frame_delay,\n                                                  \"redraw\": True},\n                                        \"fromcurrent\": True,\n                                        \"transition\": {\"duration\": 0}}],\n                        \"label\": \"&#9654;\",\n                        \"method\": \"animate\",\n                    },\n                    {\n                        \"args\": [[None], {\"frame\": {\"duration\": 0, \"redraw\": False},\n                                          \"mode\": \"immediate\",\n                                          \"transition\": {\"duration\": 0}}],\n                        \"label\": \"&#9612;&#9612;\",\n                        \"method\": \"animate\",\n                    },\n                ],\n                \"direction\": \"left\",\n                \"pad\": {\"r\": 100, \"t\": 100},\n                \"font\": {\"size\":20},\n                \"type\": \"buttons\",\n                \"x\": 0.1,\n                \"y\": 0,\n            }\n        ],\n    )\n    camera = dict(\n        up=dict(x=0, y=-1, z=0),\n        eye=dict(x=0, y=0, z=2.5)\n    )\n    fig.update_layout(title_text=title, title_x=0.5)\n    fig.update_layout(scene_camera=camera, showlegend=False)\n    fig.update_layout(xaxis = dict(visible=False),\n            yaxis = dict(visible=False),\n    )\n    fig.update_yaxes(autorange=\"reversed\")\n\n    fig.show()\n    \n    \ndef map_new_to_old_style(sequence):\n    types = []\n    landmark_indexes = []\n    for column in list(sequence.columns)[1:544]:\n        parts = column.split(\"_\")\n        if len(parts) == 4:\n            types.append(parts[1] + \"_\" + parts[2])\n        else:\n            types.append(parts[1])\n\n        landmark_indexes.append(int(parts[-1]))\n\n    data = {\n        \"frame\": [],\n        \"type\": [],\n        \"landmark_index\": [],\n        \"x\": [],\n        \"y\": [],\n        \"z\": []\n    }\n\n    for index, row in sequence.iterrows():\n        data[\"frame\"] += [int(row.frame)]*543\n        data[\"type\"] += types\n        data[\"landmark_index\"] += landmark_indexes\n\n        for _type, landmark_index in zip(types, landmark_indexes):\n            data[\"x\"].append(row[f\"x_{_type}_{landmark_index}\"])\n            data[\"y\"].append(row[f\"y_{_type}_{landmark_index}\"])\n            data[\"z\"].append(row[f\"z_{_type}_{landmark_index}\"])\n\n    return pd.DataFrame.from_dict(data)\n\n# assign desired colors to landmarks\ndef assign_color(row):\n    if row == 'face':\n        return 'red'\n    elif 'hand' in row:\n        return 'dodgerblue'\n    else:\n        return 'green'\n\n# specifies the plotting order\ndef assign_order(row):\n    if row.type == 'face':\n        return row.landmark_index + 101\n    elif row.type == 'pose':\n        return row.landmark_index + 30\n    elif row.type == 'left_hand':\n        return row.landmark_index + 80\n    else:\n        return row.landmark_index\n    \n    \ndef get_phrase(df, file_id, sequence_id):\n    return df[\n        np.logical_and(\n            df.file_id == file_id, \n            df.sequence_id == sequence_id\n        )\n    ].phrase.iloc[0]","metadata":{"execution":{"iopub.status.busy":"2023-06-13T05:55:19.555411Z","iopub.execute_input":"2023-06-13T05:55:19.555818Z","iopub.status.idle":"2023-06-13T05:55:19.580068Z","shell.execute_reply.started":"2023-06-13T05:55:19.555786Z","shell.execute_reply":"2023-06-13T05:55:19.579113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data EDA","metadata":{}},{"cell_type":"code","source":"BASE_DIR = \"../input/asl-fingerspelling\"\ntrain = pd.read_csv(f'{BASE_DIR}/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-13T05:55:23.533906Z","iopub.execute_input":"2023-06-13T05:55:23.534291Z","iopub.status.idle":"2023-06-13T05:55:23.597369Z","shell.execute_reply.started":"2023-06-13T05:55:23.534266Z","shell.execute_reply":"2023-06-13T05:55:23.595856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()\n\n#train.csv 1. 有每一个对应parquet的path; 2.file_id, sequence_id, participant_id和phrase.","metadata":{"execution":{"iopub.status.busy":"2023-06-13T05:55:24.955723Z","iopub.execute_input":"2023-06-13T05:55:24.956089Z","iopub.status.idle":"2023-06-13T05:55:24.977713Z","shell.execute_reply.started":"2023-06-13T05:55:24.956042Z","shell.execute_reply":"2023-06-13T05:55:24.975714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 我们要预测什么？\n\n- 46478 unique phrases\n- Ranging from 1-17 examples of each phrases","metadata":{}},{"cell_type":"code","source":"train['phrase'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T02:55:35.483206Z","iopub.execute_input":"2023-06-13T02:55:35.483661Z","iopub.status.idle":"2023-06-13T02:55:35.538129Z","shell.execute_reply.started":"2023-06-13T02:55:35.483628Z","shell.execute_reply":"2023-06-13T02:55:35.537308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['phrase'].value_counts().head(20)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T02:58:23.547754Z","iopub.execute_input":"2023-06-13T02:58:23.548363Z","iopub.status.idle":"2023-06-13T02:58:23.599773Z","shell.execute_reply.started":"2023-06-13T02:58:23.548316Z","shell.execute_reply":"2023-06-13T02:58:23.598466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parquet Landmark Data","metadata":{}},{"cell_type":"markdown","source":"- sequence_id - 地标序列的唯一标识符。地标文件包含大约1000个序列。序列ID被用作数据框架的索引。\n\n- frame - 一个地标序列中的帧号。\n\n- [x/y/z]_[type]_[landmark_index], 现在有1,629个空间坐标列，用于543个地标中每个地标的x、y和z坐标。\n\n- 地标的类型是 ['face', 'left_hand', 'pose', 'right_hand'].中的一个。关于手部地标位置的细节可以在这里找到。空间坐标已经被MediaPipe规范化了。请注意，MediaPipe模型还没有完全训练好预测深度，所以你可能希望忽略Z值。地标已经被转换为float32。","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Is there only one phrase per parquet file?\n- NO, each parquet file has many phrases","metadata":{}},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2023-06-13T03:09:23.837622Z","iopub.execute_input":"2023-06-13T03:09:23.838116Z","iopub.status.idle":"2023-06-13T03:09:23.855849Z","shell.execute_reply.started":"2023-06-13T03:09:23.838054Z","shell.execute_reply":"2023-06-13T03:09:23.854699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"path\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T05:55:32.197749Z","iopub.execute_input":"2023-06-13T05:55:32.198158Z","iopub.status.idle":"2023-06-13T05:55:32.215834Z","shell.execute_reply.started":"2023-06-13T05:55:32.198128Z","shell.execute_reply":"2023-06-13T05:55:32.215099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_fn = train.query('phrase == \"3 creekhouse\"')[\"path\"].values[0]\nexample_landmark = pd.read_parquet(f'{BASE_DIR}/{example_fn}')","metadata":{"execution":{"iopub.status.busy":"2023-06-13T05:55:34.315062Z","iopub.execute_input":"2023-06-13T05:55:34.315472Z","iopub.status.idle":"2023-06-13T05:55:47.834026Z","shell.execute_reply.started":"2023-06-13T05:55:34.315442Z","shell.execute_reply":"2023-06-13T05:55:47.833185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"phrase_string = train.loc[train[\"sequence_id\"] == 1816796431].phrase.values[0]","metadata":{"execution":{"iopub.status.busy":"2023-06-13T06:06:50.542593Z","iopub.execute_input":"2023-06-13T06:06:50.542967Z","iopub.status.idle":"2023-06-13T06:06:50.549482Z","shell.execute_reply.started":"2023-06-13T06:06:50.542937Z","shell.execute_reply":"2023-06-13T06:06:50.548348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T06:02:16.90772Z","iopub.execute_input":"2023-06-13T06:02:16.908164Z","iopub.status.idle":"2023-06-13T06:02:16.937849Z","shell.execute_reply.started":"2023-06-13T06:02:16.908132Z","shell.execute_reply":"2023-06-13T06:02:16.936269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark=example_landmark.reset_index(inplace=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T06:00:00.612531Z","iopub.execute_input":"2023-06-13T06:00:00.613123Z","iopub.status.idle":"2023-06-13T06:00:00.976063Z","shell.execute_reply.started":"2023-06-13T06:00:00.613037Z","shell.execute_reply":"2023-06-13T06:00:00.97442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# view number of unique sequence_ids in dataset \nexample_landmark[\"sequence_id\"].nunique() # 1000","metadata":{"execution":{"iopub.status.busy":"2023-06-13T06:00:03.364137Z","iopub.execute_input":"2023-06-13T06:00:03.364598Z","iopub.status.idle":"2023-06-13T06:00:03.378858Z","shell.execute_reply.started":"2023-06-13T06:00:03.36456Z","shell.execute_reply":"2023-06-13T06:00:03.377248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# return 1st two sequence_ids\nexample_landmark[\"sequence_id\"].unique()[:2]","metadata":{"execution":{"iopub.status.busy":"2023-06-13T06:00:51.959816Z","iopub.execute_input":"2023-06-13T06:00:51.96021Z","iopub.status.idle":"2023-06-13T06:00:51.969807Z","shell.execute_reply.started":"2023-06-13T06:00:51.960186Z","shell.execute_reply":"2023-06-13T06:00:51.9684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_phrase_1st = example_landmark[example_landmark[\"sequence_id\"]==1816796431]\ntarget_phrase_1st","metadata":{"execution":{"iopub.status.busy":"2023-06-13T06:03:44.479657Z","iopub.execute_input":"2023-06-13T06:03:44.480137Z","iopub.status.idle":"2023-06-13T06:03:44.505402Z","shell.execute_reply.started":"2023-06-13T06:03:44.480101Z","shell.execute_reply":"2023-06-13T06:03:44.504723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualise2d_landmarks(target_phrase_1st, f\"Phrase: {phrase_string}\", inter_frame_delay=200)\nvisualise2d_landmarks(target_phrase_1st, f\"Phrase: {phrase_string}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-13T03:26:51.309873Z","iopub.execute_input":"2023-06-13T03:26:51.310343Z","iopub.status.idle":"2023-06-13T03:27:06.081541Z","shell.execute_reply.started":"2023-06-13T03:26:51.310307Z","shell.execute_reply":"2023-06-13T03:27:06.080188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ndef load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:03:34.870216Z","iopub.execute_input":"2023-06-13T09:03:34.870608Z","iopub.status.idle":"2023-06-13T09:03:34.875644Z","shell.execute_reply.started":"2023-06-13T09:03:34.87058Z","shell.execute_reply":"2023-06-13T09:03:34.874521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '../input/asl-fingerspelling/train_landmarks/5414471.parquet'\nframes = load_relevant_data_subset(path)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:03:36.821399Z","iopub.execute_input":"2023-06-13T09:03:36.821787Z","iopub.status.idle":"2023-06-13T09:03:39.521064Z","shell.execute_reply.started":"2023-06-13T09:03:36.821759Z","shell.execute_reply":"2023-06-13T09:03:39.519904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tf.lite.Interpreter('../input/modeltest/model.tflite')\n\nREQUIRED_SIGNATURE = \"serving_default\"\nREQUIRED_OUTPUT = \"outputs\"\n\nwith open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    character_map = json.load(f)\nrev_character_map = {j:i for i,j in character_map.items()}\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\nif REQUIRED_SIGNATURE not in found_signatures:\n    raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=frames)\nprediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\nprint(\"pred =\", prediction_str)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:03:12.379184Z","iopub.execute_input":"2023-06-13T09:03:12.379639Z","iopub.status.idle":"2023-06-13T09:03:12.477858Z","shell.execute_reply.started":"2023-06-13T09:03:12.379606Z","shell.execute_reply":"2023-06-13T09:03:12.476259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:58:27.901685Z","iopub.execute_input":"2023-06-13T08:58:27.902109Z","iopub.status.idle":"2023-06-13T08:58:27.909586Z","shell.execute_reply.started":"2023-06-13T08:58:27.902079Z","shell.execute_reply":"2023-06-13T08:58:27.908426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Pre-processing","metadata":{}},{"cell_type":"code","source":"BASE_DIR = \"../input/asl-fingerspelling\"\ntrain = pd.read_csv(f'{BASE_DIR}/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:08:04.977187Z","iopub.execute_input":"2023-06-13T09:08:04.977651Z","iopub.status.idle":"2023-06-13T09:08:05.134297Z","shell.execute_reply.started":"2023-06-13T09:08:04.977615Z","shell.execute_reply":"2023-06-13T09:08:05.132954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\nnum_to_char = {j:i for i,j in char_to_num.items()}\n\n# inpdir = \"/kaggle/input/asl-fingerspelling\"\n# df = pd.read_csv(f'{inpdir}/train.csv')\n\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nRHAND_LBLS = [f'x_right_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)] + [f'z_right_hand_{i}' for i in range(21)]\nLHAND_LBLS = [ f'x_left_hand_{i}' for i in range(21)] + [ f'y_left_hand_{i}' for i in range(21)] + [ f'z_left_hand_{i}' for i in range(21)]\nPOSE_LBLS = [f'x_pose_{i}' for i in POSE] + [f'y_pose_{i}' for i in POSE] + [f'z_pose_{i}' for i in POSE]\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE]\n\nSEL_COLS = X + Y + Z\nFRAME_LEN = 128\n\nX_IDX = [i for i, col in enumerate(SEL_COLS)  if \"x_\" in col]\nY_IDX = [i for i, col in enumerate(SEL_COLS)  if \"y_\" in col]\nZ_IDX = [i for i, col in enumerate(SEL_COLS)  if \"z_\" in col]\n\nRHAND_IDX = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col]\nLHAND_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col]\nRPOSE_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE]\nLPOSE_IDX = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE]","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:08:06.703164Z","iopub.execute_input":"2023-06-13T09:08:06.703581Z","iopub.status.idle":"2023-06-13T09:08:06.722328Z","shell.execute_reply.started":"2023-06-13T09:08:06.703545Z","shell.execute_reply":"2023-06-13T09:08:06.721169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_pad(x):\n    if tf.shape(x)[0] < FRAME_LEN:\n        x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]))\n    else:\n        x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n    return x\n\ndef pre_process(x):\n    rhand = tf.gather(x, RHAND_IDX, axis=1)\n    lhand = tf.gather(x, LHAND_IDX, axis=1)\n    rpose = tf.gather(x, RPOSE_IDX, axis=1)\n    lpose = tf.gather(x, LPOSE_IDX, axis=1)\n    \n    rnan_idx = tf.reduce_any(tf.math.is_nan(rhand), axis=1)\n    lnan_idx = tf.reduce_any(tf.math.is_nan(lhand), axis=1)\n    \n    rnans = tf.math.count_nonzero(rnan_idx)\n    lnans = tf.math.count_nonzero(lnan_idx)\n    \n    # For dominant hand\n    if rnans > lnans:\n        hand = lhand\n        pose = lpose\n        \n        hand_x = hand[:, 0*(len(LHAND_IDX)//3) : 1*(len(LHAND_IDX)//3)]\n        hand_y = hand[:, 1*(len(LHAND_IDX)//3) : 2*(len(LHAND_IDX)//3)]\n        hand_z = hand[:, 2*(len(LHAND_IDX)//3) : 3*(len(LHAND_IDX)//3)]\n        hand = tf.concat([1-hand_x, hand_y, hand_z], axis=1)\n        \n        pose_x = pose[:, 0*(len(LPOSE_IDX)//3) : 1*(len(LPOSE_IDX)//3)]\n        pose_y = pose[:, 1*(len(LPOSE_IDX)//3) : 2*(len(LPOSE_IDX)//3)]\n        pose_z = pose[:, 2*(len(LPOSE_IDX)//3) : 3*(len(LPOSE_IDX)//3)]\n        pose = tf.concat([1-pose_x, pose_y, pose_z], axis=1)\n    else:\n        hand = rhand\n        pose = rpose\n    \n    hand_x = hand[:, 0*(len(LHAND_IDX)//3) : 1*(len(LHAND_IDX)//3)]\n    hand_y = hand[:, 1*(len(LHAND_IDX)//3) : 2*(len(LHAND_IDX)//3)]\n    hand_z = hand[:, 2*(len(LHAND_IDX)//3) : 3*(len(LHAND_IDX)//3)]\n    hand = tf.concat([hand_x[..., tf.newaxis], hand_y[..., tf.newaxis], hand_z[..., tf.newaxis]], axis=-1)\n    \n    mean = tf.math.reduce_mean(hand, axis=1)[:, tf.newaxis, :]\n    std = tf.math.reduce_std(hand, axis=1)[:, tf.newaxis, :]\n    hand = (hand - mean) / std\n\n    pose_x = pose[:, 0*(len(LPOSE_IDX)//3) : 1*(len(LPOSE_IDX)//3)]\n    pose_y = pose[:, 1*(len(LPOSE_IDX)//3) : 2*(len(LPOSE_IDX)//3)]\n    pose_z = pose[:, 2*(len(LPOSE_IDX)//3) : 3*(len(LPOSE_IDX)//3)]\n    pose = tf.concat([pose_x[..., tf.newaxis], pose_y[..., tf.newaxis], pose_z[..., tf.newaxis]], axis=-1)\n    \n    x = tf.concat([hand, pose], axis=1)\n    x = resize_pad(x)\n    \n    x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n    x = tf.reshape(x, (FRAME_LEN, len(LHAND_IDX) + len(LPOSE_IDX)))\n    return x\n\ndef load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=SEL_COLS)\n\n\nfile_id = train.file_id.iloc[0] #file_id = 5414471\n\npqfile = f\"{BASE_DIR}/train_landmarks/{file_id}.parquet\" #找到对应路径\n\nseq_refs = train.loc[train.file_id == file_id] # 找到所有ile_id = 5414471的数据\nseqs = load_relevant_data_subset(pqfile) #加载 5414471.parquet\n\nseq_id = seq_refs.sequence_id.iloc[0] #seq_id = 1816796431\nframes = seqs.iloc[seqs.index == seq_id] #找到所有 seq_id = 1816796431的数据 （3 creekhouse 对应的所有数据）\nphrase = str(train.loc[train.sequence_id == seq_id].phrase.iloc[0]) #对应的phrase\n\n# file_id = df.file_id.iloc[0]\n# inpdir = \"/kaggle/input/asl-fingerspelling\"\n# pqfile = f\"{inpdir}/train_landmarks/{file_id}.parquet\"\n# seq_refs = df.loc[df.file_id == file_id]\n# seqs = load_relevant_data_subset(pqfile)\n\n# seq_id = seq_refs.sequence_id.iloc[0]\n# frames = seqs.iloc[seqs.index == seq_id]\n# phrase = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])\n\nprint(pre_process(frames).shape)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:08:16.303681Z","iopub.execute_input":"2023-06-13T09:08:16.304108Z","iopub.status.idle":"2023-06-13T09:08:16.678365Z","shell.execute_reply.started":"2023-06-13T09:08:16.304075Z","shell.execute_reply":"2023-06-13T09:08:16.677238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table = tf.lookup.StaticHashTable(\n    initializer=tf.lookup.KeyValueTensorInitializer(\n        keys=list(char_to_num.keys()),\n        values=list(char_to_num.values()),\n    ),\n    default_value=tf.constant(-1),\n    name=\"class_weight\"\n)\n\nmask_idx = char_to_num['#']\n\ndef preprocess_fn(landmarks, phrase):\n    phrase = ';' + phrase + '['\n    phrase = tf.strings.bytes_split(phrase)\n    phrase = table.lookup(phrase)\n    phrase = tf.pad(phrase, paddings=[[0, 64 - tf.shape(phrase)[0]]], constant_values=mask_idx)\n    return pre_process(landmarks), phrase\n\ndef decode_fn(record_bytes):\n    schema = {COL: tf.io.VarLenFeature(dtype=tf.float32) for COL in SEL_COLS}\n    schema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n    features = tf.io.parse_single_example(record_bytes, schema)\n    phrase = features[\"phrase\"]\n    landmarks = ([tf.sparse.to_dense(features[COL]) for COL in SEL_COLS])\n    landmarks = tf.transpose(landmarks)\n    return landmarks, phrase\n\ninpdir = \"/kaggle/input/aslfr-preprocess-dataset\"\ntffiles = train.file_id.map(lambda x: f'{inpdir}/tfds/{x}.tfrecord').unique()\n\nbatch_size = 32\nval_len = int(0.1 * len(tffiles))\n\ntrain_dataset = tf.data.TFRecordDataset(tffiles[val_len:]).map(decode_fn).map(preprocess_fn).shuffle(buffer_size=500).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)\nval_dataset = tf.data.TFRecordDataset(tffiles[:val_len]).map(decode_fn).map(preprocess_fn).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)\ntest_dataset = tf.data.TFRecordDataset(tffiles).map(decode_fn).prefetch(buffer_size=tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:08:24.133532Z","iopub.execute_input":"2023-06-13T09:08:24.133978Z","iopub.status.idle":"2023-06-13T09:08:26.404545Z","shell.execute_reply.started":"2023-06-13T09:08:24.133943Z","shell.execute_reply":"2023-06-13T09:08:26.403348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inpdir = \"/kaggle/input/asl-fingerspelling\"\ndf = pd.read_csv(f'{inpdir}/train.csv')\ndf[\"phrase_bytes\"] = df[\"phrase\"].map(lambda x: x.encode(\"utf-8\"))\ndisplay(df.head())","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:10:00.867121Z","iopub.execute_input":"2023-06-13T09:10:00.867604Z","iopub.status.idle":"2023-06-13T09:10:01.009247Z","shell.execute_reply.started":"2023-06-13T09:10:00.867567Z","shell.execute_reply":"2023-06-13T09:10:01.008053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LIP = [\n    61, 185, 40, 39, 37, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\n\nFACE = [f'x_face_{i}' for i in LIP] + [f'y_face_{i}' for i in LIP] + [f'z_face_{i}' for i in LIP]\nLHAND = [f'x_left_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)]\nRHAND = [f'x_right_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)] + [f'z_right_hand_{i}' for i in range(21)]\nPOSE = [f'x_pose_{i}' for i in range(33)] + [f'y_pose_{i}' for i in range(33)] + [f'z_pose_{i}' for i in range(33)]\n\nSEL_COLS = FACE + LHAND + RHAND + POSE\nFRAME_LEN = 128","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:10:12.55053Z","iopub.execute_input":"2023-06-13T09:10:12.551184Z","iopub.status.idle":"2023-06-13T09:10:12.563284Z","shell.execute_reply.started":"2023-06-13T09:10:12.551142Z","shell.execute_reply":"2023-06-13T09:10:12.561824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_id = '5414471'\ntestfile = f\"../input/asl-fingerspelling/train_landmarks/{file_id}.parquet\"\n\ntffile = f\"{file_id}.tfrecord\"\nseq_refs = df.loc[df.file_id == file_id]\nseqs = load_relevant_data_subset(testfile)\n    \nwith tf.io.TFRecordWriter(tffile) as file_writer:\n    for seq_id, phrase in zip(seq_refs.sequence_id, seq_refs.phrase_bytes):\n        frames = seqs.iloc[seqs.index == seq_id]\n        frames128 = frames.fillna(-10).to_numpy()\n        frames128 = resize(frames128, (FRAME_LEN, len(SEL_COLS)))\n        frames = pd.DataFrame(data = frames128, columns=frames.columns)\n\n        features = {COL: tf.train.Feature(float_list=tf.train.FloatList(value=frames[COL])) for COL in SEL_COLS}\n        features[\"phrase\"] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[phrase]))\n        record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n        file_writer.write(record_bytes)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:38:00.838484Z","iopub.execute_input":"2023-06-13T09:38:00.838897Z","iopub.status.idle":"2023-06-13T09:38:01.479677Z","shell.execute_reply.started":"2023-06-13T09:38:00.838866Z","shell.execute_reply":"2023-06-13T09:38:01.478366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset_path = '../output/tfds/5414471.tfrecord'","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:45:07.192996Z","iopub.execute_input":"2023-06-13T09:45:07.193384Z","iopub.status.idle":"2023-06-13T09:45:07.19899Z","shell.execute_reply.started":"2023-06-13T09:45:07.193356Z","shell.execute_reply":"2023-06-13T09:45:07.197528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = tf.data.TFRecordDataset(test_dataset_path)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:45:10.428756Z","iopub.execute_input":"2023-06-13T09:45:10.429161Z","iopub.status.idle":"2023-06-13T09:45:10.449064Z","shell.execute_reply.started":"2023-06-13T09:45:10.429133Z","shell.execute_reply":"2023-06-13T09:45:10.448051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:45:14.621177Z","iopub.execute_input":"2023-06-13T09:45:14.621648Z","iopub.status.idle":"2023-06-13T09:45:14.629585Z","shell.execute_reply.started":"2023-06-13T09:45:14.621611Z","shell.execute_reply":"2023-06-13T09:45:14.628394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for frame, target in test_dataset.skip(1000).take(10):\n    ","metadata":{"execution":{"iopub.status.busy":"2023-06-13T09:13:36.400071Z","iopub.execute_input":"2023-06-13T09:13:36.401101Z","iopub.status.idle":"2023-06-13T09:13:58.790608Z","shell.execute_reply.started":"2023-06-13T09:13:36.401064Z","shell.execute_reply":"2023-06-13T09:13:58.789022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_fn(record_bytes):\n    schema = {COL: tf.io.FixedLenFeature([FRAME_LEN], dtype=tf.float32) for COL in SEL_COLS}\n    schema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n    return tf.io.parse_single_example(record_bytes, schema)\n\nfor file_id in df.file_id:\n    pqfile = f\"{inpdir}/train_landmarks/{file_id}.parquet\"\n    if not os.path.isdir(\"tfds\"): os.mkdir(\"tfds\")\n    tffile = f\"tfds/{file_id}.tfrecord\"\n    for batch in tf.data.TFRecordDataset([tffile]).map(decode_fn).take(2):\n        print(list(batch.keys())[0])\n    break","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = prediction_fn(inputs = frames)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T10:05:49.766574Z","iopub.execute_input":"2023-06-13T10:05:49.767174Z","iopub.status.idle":"2023-06-13T10:05:49.841808Z","shell.execute_reply.started":"2023-06-13T10:05:49.767124Z","shell.execute_reply":"2023-06-13T10:05:49.839882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction['outputs']","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:23:39.380855Z","iopub.execute_input":"2023-06-13T08:23:39.38146Z","iopub.status.idle":"2023-06-13T08:23:39.390557Z","shell.execute_reply.started":"2023-06-13T08:23:39.381416Z","shell.execute_reply":"2023-06-13T08:23:39.389287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:18:04.33737Z","iopub.execute_input":"2023-06-13T08:18:04.337767Z","iopub.status.idle":"2023-06-13T08:18:04.346183Z","shell.execute_reply.started":"2023-06-13T08:18:04.337737Z","shell.execute_reply":"2023-06-13T08:18:04.344984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(test_dataset)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:20:00.488129Z","iopub.execute_input":"2023-06-13T08:20:00.488557Z","iopub.status.idle":"2023-06-13T08:22:16.709581Z","shell.execute_reply.started":"2023-06-13T08:20:00.488523Z","shell.execute_reply":"2023-06-13T08:22:16.706878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(test_dataset.skip(10))","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:31:39.933447Z","iopub.execute_input":"2023-06-13T08:31:39.933913Z","iopub.status.idle":"2023-06-13T08:31:50.935754Z","shell.execute_reply.started":"2023-06-13T08:31:39.933879Z","shell.execute_reply":"2023-06-13T08:31:50.934063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for frame, target in test_dataset.skip(1000).take(10):\n    print(frame)\n    print(target)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:39:26.892097Z","iopub.execute_input":"2023-06-13T08:39:26.892519Z","iopub.status.idle":"2023-06-13T08:39:27.725527Z","shell.execute_reply.started":"2023-06-13T08:39:26.892486Z","shell.execute_reply":"2023-06-13T08:39:27.72413Z"},"trusted":true},"execution_count":null,"outputs":[]}]}