{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"原始英文版本：[https://www.kaggle.com/code/markwijkhuizen/aslfr-eda-preprocessing-dataset](https://www.kaggle.com/code/markwijkhuizen/aslfr-eda-preprocessing-dataset)","metadata":{}},{"cell_type":"markdown","source":"# 导入库","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\nimport tensorflow as tf\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit\nfrom pathlib import Path\n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport time\nimport json\nimport re\n\n# TQDM Progress Bar With Pandas Apply Function\ntqdm.pandas()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:36:09.11245Z","iopub.execute_input":"2023-06-19T06:36:09.112876Z","iopub.status.idle":"2023-06-19T06:36:20.079136Z","shell.execute_reply.started":"2023-06-19T06:36:09.112845Z","shell.execute_reply":"2023-06-19T06:36:20.077853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Character To Ordinal Encoding","metadata":{}},{"cell_type":"code","source":"# Read Character to Ordinal Encoding Mapping\n# 读取字符编码json\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    CHAR2ORD = json.load(json_file)\n    \n# Character to Ordinal Encoding Mapping   \n# 展示字符编码json\ndisplay(pd.Series(CHAR2ORD).to_frame('Ordinal Encoding'))","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:36:23.12671Z","iopub.execute_input":"2023-06-19T06:36:23.127538Z","iopub.status.idle":"2023-06-19T06:36:23.175704Z","shell.execute_reply.started":"2023-06-19T06:36:23.127498Z","shell.execute_reply":"2023-06-19T06:36:23.174464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of Unique Characters\n# 总共字符个数\nN_UNIQUE_CHARACTERS = len(CHAR2ORD)\nprint(f'N_UNIQUE_CHARACTERS: {N_UNIQUE_CHARACTERS}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:36:39.637338Z","iopub.execute_input":"2023-06-19T06:36:39.637761Z","iopub.status.idle":"2023-06-19T06:36:39.644675Z","shell.execute_reply.started":"2023-06-19T06:36:39.637717Z","shell.execute_reply":"2023-06-19T06:36:39.643194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 全局配置","metadata":{}},{"cell_type":"code","source":"# If Notebook Is Run By Committing or In Interactive Mode For Development\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n\n# Describe Statistics Percentiles\n# 统计分位点\nPERCENTILES = [0.01, 0.10, 0.05, 0.25, 0.50, 0.75, 0.90, 0.95, 0.99, 0.999]\n\n# Global Random Seed\n# 全局随机数\nSEED = 42\n\n# Number of Frames to resize recording to\n# 设置的帧个数\nN_TARGET_FRAMES = 128\n\n# Global debug flag, takes subset of train\n# 设置是否debug模式，选择部分数据进行运行代码\nDEBUG = False\n\n# Fast Processing\nFAST = False\n\n# Number of Unique Characters To Predict + Pad Token + SOS Token + EOS Token\n# 需要预测的字符个数\nN_UNIQUE_CHARACTERSPAD_TOKEN = len(CHAR2ORD)\nSOS_TOKEN = len(CHAR2ORD) + 1 # Start Of Sentence，开始字符\nEOS_TOKEN = len(CHAR2ORD) + 2 # End Of Sentence，结束字符","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:41:47.991503Z","iopub.execute_input":"2023-06-19T06:41:47.991933Z","iopub.status.idle":"2023-06-19T06:41:48.00046Z","shell.execute_reply.started":"2023-06-19T06:41:47.991897Z","shell.execute_reply":"2023-06-19T06:41:47.998928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 画图设置","metadata":{}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:41:48.876467Z","iopub.execute_input":"2023-06-19T06:41:48.877436Z","iopub.status.idle":"2023-06-19T06:41:48.886753Z","shell.execute_reply.started":"2023-06-19T06:41:48.877378Z","shell.execute_reply":"2023-06-19T06:41:48.884909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prints Shape and Dtype For List Of Variables\ndef print_shape_dtype(l, names):\n    for e, n in zip(l, names):\n        print(f'{n} shape: {e.shape}, dtype: {e.dtype}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:41:49.376643Z","iopub.execute_input":"2023-06-19T06:41:49.377023Z","iopub.status.idle":"2023-06-19T06:41:49.38262Z","shell.execute_reply.started":"2023-06-19T06:41:49.376991Z","shell.execute_reply":"2023-06-19T06:41:49.381379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 读取训练集","metadata":{}},{"cell_type":"code","source":"# Read Train DataFrame\n# 读取部分数据集 or 全部数据集\nif DEBUG:\n    train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv').head(5000)\nelse:\n    train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n\n# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-fingerspelling/{path}'\n\ntrain['file_path'] = train['path'].apply(get_file_path)\n    \n# Number Of Train Samples\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}')\n\ndisplay(train.info())\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:41:50.551605Z","iopub.execute_input":"2023-06-19T06:41:50.552804Z","iopub.status.idle":"2023-06-19T06:41:50.872078Z","shell.execute_reply.started":"2023-06-19T06:41:50.55276Z","shell.execute_reply":"2023-06-19T06:41:50.870889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Phrase类别划分","metadata":{}},{"cell_type":"code","source":"\"\"\"\nAttempt to retrieve phrase type\nCould be used for pretraining or type specific inference\n *) Phone Number\\\n *) URL\n *3) Addres\n\"\"\"\n# 对phrase进行类别划分：phone number、url、address\ndef get_phrase_type(phrase):\n    # Phone Number\n    if re.match(r'^[\\d+-]+$', phrase):\n        return 'phone_number'\n    # url\n    elif any([substr in phrase for substr in ['www', '.', '/']]) and ' ' not in phrase:\n        return 'url'\n    # Address\n    else:\n        return 'address'\n    \ntrain['phrase_type'] = train['phrase'].apply(get_phrase_type)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:41:51.791409Z","iopub.execute_input":"2023-06-19T06:41:51.791854Z","iopub.status.idle":"2023-06-19T06:41:51.996348Z","shell.execute_reply.started":"2023-06-19T06:41:51.791819Z","shell.execute_reply":"2023-06-19T06:41:51.994792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Phrase预处理","metadata":{}},{"cell_type":"code","source":"# Split Phrase To Char Tuple\n# 将phrase切分为字符\ntrain['phrase_char'] = train['phrase'].apply(tuple)\n\n# Character Length of Phrase\n# 统计phrase字符个数\ntrain['phrase_char_len'] = train['phrase_char'].apply(len)\n\n# Maximum Input Length\n# phrase最大长度\nMAX_PHRASE_LENGTH = train['phrase_char_len'].max()\nprint(f'MAX_PHRASE_LENGTH: {MAX_PHRASE_LENGTH}')\n\n# Train DataFrame indexed by sequence_id to convenientlyy lookup recording data\n# 将训练集index设置为sequence_id\ntrain_sequence_id = train.set_index('sequence_id')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:41:53.156614Z","iopub.execute_input":"2023-06-19T06:41:53.157028Z","iopub.status.idle":"2023-06-19T06:41:53.30036Z","shell.execute_reply.started":"2023-06-19T06:41:53.156997Z","shell.execute_reply":"2023-06-19T06:41:53.298411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Phrase Character Length Statistics\n# 统计不同分位点下phrase的统计\ndisplay(train['phrase_char_len'].describe(percentiles=PERCENTILES).to_frame().round(1))","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:41:54.266439Z","iopub.execute_input":"2023-06-19T06:41:54.266884Z","iopub.status.idle":"2023-06-19T06:41:54.29677Z","shell.execute_reply.started":"2023-06-19T06:41:54.266847Z","shell.execute_reply":"2023-06-19T06:41:54.294919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Character Count Occurance\n# 可视化每个长度phrase的sequence个数，大部分的phrase长度在12附近\nplt.figure(figsize=(15,8))\nplt.title('Character Length Occurance of Phrases')\ntrain['phrase_char_len'].value_counts().sort_index().plot(kind='bar')\nplt.xlim(-0.50, train['phrase_char_len'].max() - 1.50)\nplt.xlabel('Pharse Character Length')\nplt.ylabel('Sample Count')\nplt.grid(axis='y')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:41:56.141573Z","iopub.execute_input":"2023-06-19T06:41:56.142617Z","iopub.status.idle":"2023-06-19T06:41:56.765213Z","shell.execute_reply.started":"2023-06-19T06:41:56.142571Z","shell.execute_reply":"2023-06-19T06:41:56.763752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 统计phrase中字符","metadata":{}},{"cell_type":"code","source":"# Use Set to keep track of unique characters in phrases\nUNIQUE_CHARACTERS = set()\n\nfor phrase in tqdm(train['phrase_char']):\n    for c in phrase:\n        UNIQUE_CHARACTERS.add(c)\n        \n# Sorted Unique Character\nUNIQUE_CHARACTERS = np.array(sorted(UNIQUE_CHARACTERS))\n# Number of Unique Characters\nN_UNIQUE_CHARACTERS = len(UNIQUE_CHARACTERS)\nprint(f'N_UNIQUE_CHARACTERS: {N_UNIQUE_CHARACTERS}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:47:18.404374Z","iopub.execute_input":"2023-06-19T06:47:18.404916Z","iopub.status.idle":"2023-06-19T06:47:18.686789Z","shell.execute_reply.started":"2023-06-19T06:47:18.40488Z","shell.execute_reply":"2023-06-19T06:47:18.685149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 读取 Parquet 文件","metadata":{}},{"cell_type":"code","source":"# Read First Parquet File\n# 读取Parquet文件\nexample_parquet_df = pd.read_parquet(train['file_path'][0])\n\n# Display DataFrame layout\n# 展示Parquet文件内容\ndisplay(example_parquet_df.head())","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:48:44.363204Z","iopub.execute_input":"2023-06-19T06:48:44.363635Z","iopub.status.idle":"2023-06-19T06:48:47.270453Z","shell.execute_reply.started":"2023-06-19T06:48:44.3636Z","shell.execute_reply":"2023-06-19T06:48:47.26947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Parquet 文件统计","metadata":{}},{"cell_type":"code","source":"# Number of parquet chunks to analyse\n# 使用多少个Parquet进行统计\nN = 5 if IS_INTERACTIVE else 25\n\n# Number of Unique Frames in Recording\n# 存储每个Parquet中的frame id\nN_UNIQUE_FRAMES = []\n\nUNIQUE_FILE_PATHS = pd.Series(train['file_path'].unique())\n\nfor idx, file_path in enumerate(tqdm(UNIQUE_FILE_PATHS.sample(N, random_state=SEED))):\n    df = pd.read_parquet(file_path)\n    for group, group_df in df.groupby('sequence_id'):\n        N_UNIQUE_FRAMES.append(group_df['frame'].nunique())\n\n# Convert to Numpy Array\nN_UNIQUE_FRAMES = np.array(N_UNIQUE_FRAMES)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:49:02.463605Z","iopub.execute_input":"2023-06-19T06:49:02.464063Z","iopub.status.idle":"2023-06-19T06:50:36.227892Z","shell.execute_reply.started":"2023-06-19T06:49:02.46401Z","shell.execute_reply":"2023-06-19T06:50:36.226029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of unique frames in each video\n# 展示每个video存储的frame个数\ndisplay(pd.Series(N_UNIQUE_FRAMES).describe(percentiles=PERCENTILES).to_frame('Value').astype(int))\n\nplt.figure(figsize=(15,8))\nplt.title('Number of Unique Frames', size=24)\npd.Series(N_UNIQUE_FRAMES).plot(kind='hist', bins=128)\nplt.grid()\nxlim = math.ceil(plt.xlim()[1])\nplt.xlim(0, xlim)\nplt.xticks(np.arange(0, xlim+50, 50))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:51:55.77199Z","iopub.execute_input":"2023-06-19T06:51:55.773761Z","iopub.status.idle":"2023-06-19T06:51:56.441312Z","shell.execute_reply.started":"2023-06-19T06:51:55.773693Z","shell.execute_reply":"2023-06-19T06:51:56.439851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# With N_TARGET_FRAMES = 256 ~85% will be below\nN_UNIQUE_FRAMES_WATERFALL = []\n\n# Maximum Number of Unique Frames to use\nN_MAX_UNIQUE_FRAMES = 400\n# Compute Percentage\nfor n in tqdm(range(0,N_MAX_UNIQUE_FRAMES+1)):\n    N_UNIQUE_FRAMES_WATERFALL.append(sum(N_UNIQUE_FRAMES >= n) / len(N_UNIQUE_FRAMES) * 100)\n\nplt.figure(figsize=(18,10))\nplt.title('Waterfall Plot For Number Of Unique Frames')\npd.Series(N_UNIQUE_FRAMES_WATERFALL).plot(kind='bar')\nplt.grid(axis='y')\nplt.xticks([1] + np.arange(5, N_MAX_UNIQUE_FRAMES+5, 5).tolist(), size=8, rotation=45)\nplt.xlabel('Number of Unique Frames', size=16)\nplt.yticks(np.arange(0, 100+5, 5), [f'{i}%' for i in range(0,100+5,5)])\nplt.ylim(0, 100)\nplt.ylabel('Percentage of Samples With At Least N Unique Frames', size=16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:52:04.822138Z","iopub.execute_input":"2023-06-19T06:52:04.822561Z","iopub.status.idle":"2023-06-19T06:52:07.572043Z","shell.execute_reply.started":"2023-06-19T06:52:04.822527Z","shell.execute_reply":"2023-06-19T06:52:07.570781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Landmark分析","metadata":{}},{"cell_type":"code","source":"'''\ndf: 原始的Parquet文件\nwords_pos: 正向关键词\nwords_neg: 负向关键词\nret_names: 是否返回列名称\nidxs_pos: 列索引\n\n函数的功能是按照正向关键词 or 列索引找到 Parquet文件的列，并剔除包含负向关键词的列\n'''\ndef get_idxs(df, words_pos, words_neg=[], ret_names=True, idxs_pos=None):\n    idxs = []\n    names = []\n    for w in words_pos:\n        for col_idx, col in enumerate(example_parquet_df.columns):\n            \n            # Exclude Non Landmark Columns\n            if col in ['frame']:\n                continue\n                \n            col_idx = int(col.split('_')[-1])\n            \n            # Check if column name contains all words\n            if (w in col) and (idxs_pos is None or col_idx in idxs_pos) and all([w not in col for w in words_neg]):\n                idxs.append(col_idx)\n                names.append(col)\n    \n    # Convert to Numpy arrays\n    idxs = np.array(idxs)\n    names = np.array(names)\n    \n    # Returns either both column indices and names\n    if ret_names:\n        return idxs, names\n    \n    # Or only columns indices\n    else:\n        return idxs","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:53:07.951955Z","iopub.execute_input":"2023-06-19T06:53:07.952439Z","iopub.status.idle":"2023-06-19T06:53:07.963986Z","shell.execute_reply.started":"2023-06-19T06:53:07.952401Z","shell.execute_reply":"2023-06-19T06:53:07.962308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lips Landmark Face Ids\nLIPS_LANDMARK_IDXS = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\n\n# 筛选包含 left_hand 但不包含 z 的列\nLEFT_HAND_IDXS0, LEFT_HAND_NAMES0 = get_idxs(example_parquet_df, ['left_hand'], ['z'])\nprint(len(LEFT_HAND_IDXS0), LEFT_HAND_NAMES0)\n\n# 筛选包含 right_hand 但不包含 z 的列\nRIGHT_HAND_IDXS0, RIGHT_HAND_NAMES0 = get_idxs(example_parquet_df, ['right_hand'], ['z'])\nprint(len(RIGHT_HAND_IDXS0), RIGHT_HAND_NAMES0)\n\n# 筛选包含 face + LIPS_LANDMARK_IDXS索引 但不包含 z 的列\nLIPS_IDXS0, LIPS_NAMES0 = get_idxs(example_parquet_df, ['face'], ['z'], idxs_pos=LIPS_LANDMARK_IDXS)\nprint(len(LIPS_IDXS0), LIPS_NAMES0)\n\n# 将筛选得到的列，进行合并\nCOLUMNS0 = np.concatenate((LEFT_HAND_NAMES0, RIGHT_HAND_NAMES0, LIPS_NAMES0))\nN_COLS0 = len(COLUMNS0)\nN_DIMS0 = 2\n\nprint(f'N_COLS0: {N_COLS0}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:04:15.617028Z","iopub.execute_input":"2023-06-19T07:04:15.617519Z","iopub.status.idle":"2023-06-19T07:04:15.651415Z","shell.execute_reply.started":"2023-06-19T07:04:15.617483Z","shell.execute_reply":"2023-06-19T07:04:15.650013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Landmark Indices in subset of dataframe with only COLUMNS selected\nLEFT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, LEFT_HAND_NAMES0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(COLUMNS0, RIGHT_HAND_NAMES0)).squeeze()\nLIPS_IDXS = np.argwhere(np.isin(COLUMNS0, LIPS_NAMES0)).squeeze()\nN_COLS = N_COLS0\n\n# Only X/Y axes are used\nN_DIMS = 2\n\nprint(LEFT_HAND_IDXS, RIGHT_HAND_IDXS, LIPS_IDXS)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:05:45.028517Z","iopub.execute_input":"2023-06-19T07:05:45.029319Z","iopub.status.idle":"2023-06-19T07:05:45.040882Z","shell.execute_reply.started":"2023-06-19T07:05:45.029261Z","shell.execute_reply":"2023-06-19T07:05:45.039689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Indices in processed data by axes with only dominant hand\n# 得到left hand的索引\nHAND_X_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'x' in name]\n    ).squeeze()\nHAND_Y_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'y' in name]\n    ).squeeze()\n# Names in processed data by axes\nHAND_X_NAMES = LEFT_HAND_NAMES0[HAND_X_IDXS]\nHAND_Y_NAMES = LEFT_HAND_NAMES0[HAND_Y_IDXS]","metadata":{"execution":{"iopub.status.busy":"2023-06-19T06:54:48.66201Z","iopub.execute_input":"2023-06-19T06:54:48.662704Z","iopub.status.idle":"2023-06-19T06:54:48.670314Z","shell.execute_reply.started":"2023-06-19T06:54:48.662667Z","shell.execute_reply":"2023-06-19T06:54:48.668758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 处理每个特征的缺失值","metadata":{}},{"cell_type":"code","source":"\"\"\"\n    Tensorflow layer to process data in TFLite\n    Data needs to be processed in the model itself, so we can not use Python\n\"\"\" \nclass PreprocessLayerNonNaN(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayerNonNaN, self).__init__()\n    \n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,N_COLS0], dtype=tf.float32),),\n    )\n    def call(self, data0):\n        # Fill NaN Values With 0\n        # 缺失值填充\n        data = tf.where(tf.math.is_nan(data0), 0.0, data0)\n        \n        # Hacky\n        # 增加数据维度\n        data = data[None]\n        \n        # Empty Hand Frame Filtering\n        # 对数据筛选\n        hands = tf.slice(data, [0,0,0], [-1, -1, 84])\n        hands = tf.abs(hands)\n        mask = tf.reduce_sum(hands, axis=2)\n        mask = tf.not_equal(mask, 0)\n        data = data[mask][None]\n        data = tf.squeeze(data, axis=[0])\n        \n        return data\n    \npreprocess_layer_non_nan = PreprocessLayerNonNaN()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:12:41.02257Z","iopub.execute_input":"2023-06-19T07:12:41.023124Z","iopub.status.idle":"2023-06-19T07:12:41.082148Z","shell.execute_reply.started":"2023-06-19T07:12:41.023082Z","shell.execute_reply":"2023-06-19T07:12:41.080382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unique Parquet Files\nUNIQUE_FILE_PATHS = pd.Series(train['file_path'].unique())\n# Number of parquet chunks to analyse\nN = 5 if (IS_INTERACTIVE or FAST) else len(UNIQUE_FILE_PATHS)\n# Number of Non Nan Frames in Recording\nN_NON_NAN_FRAMES = []\n\nfor idx, file_path in enumerate(tqdm(UNIQUE_FILE_PATHS.sample(N, random_state=SEED))):\n    df = pd.read_parquet(file_path)\n    for group, group_df in df.groupby('sequence_id'):\n        frames = preprocess_layer_non_nan(group_df[COLUMNS0].values).numpy()\n        N_NON_NAN_FRAMES.append(len(frames))\n\n# Convert to Numpy Array\n# 统计每个sequence中非空的frame个数\nN_NON_NAN_FRAMES = pd.Series(N_NON_NAN_FRAMES).to_frame('# Frames')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:12:42.761745Z","iopub.execute_input":"2023-06-19T07:12:42.762181Z","iopub.status.idle":"2023-06-19T07:14:49.962515Z","shell.execute_reply.started":"2023-06-19T07:12:42.762147Z","shell.execute_reply":"2023-06-19T07:14:49.96157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of frames in each video with hand coordinates\ndisplay(N_NON_NAN_FRAMES.describe(percentiles=PERCENTILES).astype(int))\n\nN_NON_NAN_FRAMES.plot(kind='hist', bins=128, figsize=(15,8))\nplt.title('Number of Non NaN Frames', size=24)\nplt.grid()\nxlim = np.percentile(N_NON_NAN_FRAMES, 99)\nplt.xlim(0, xlim)\nplt.xticks(np.arange(0, xlim+32, 32))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:14:58.362305Z","iopub.execute_input":"2023-06-19T07:14:58.362766Z","iopub.status.idle":"2023-06-19T07:14:59.029784Z","shell.execute_reply.started":"2023-06-19T07:14:58.362727Z","shell.execute_reply":"2023-06-19T07:14:59.028336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorflow预处理层","metadata":{}},{"cell_type":"code","source":"\"\"\"\n    Tensorflow layer to process data in TFLite\n    Data needs to be processed in the model itself, so we can not use Python\n\"\"\" \nclass PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n    \n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,N_COLS0], dtype=tf.float32),),\n    )\n    def call(self, data0, resize=True):\n        # Fill NaN Values With 0\n        data = tf.where(tf.math.is_nan(data0), 0.0, data0)\n        \n        # Hacky\n        data = data[None]\n        \n        # Empty Hand Frame Filtering\n        hands = tf.slice(data, [0,0,0], [-1, -1, 84])\n        hands = tf.abs(hands)\n        mask = tf.reduce_sum(hands, axis=2)\n        mask = tf.not_equal(mask, 0)\n        data = data[mask][None]\n        \n        # Pad Zeros\n        # 加入pad操作，保证维度相同\n        N_FRAMES = len(data[0])\n        if N_FRAMES < N_TARGET_FRAMES:\n            data = tf.concat((\n                data,\n                tf.zeros([1,N_TARGET_FRAMES-N_FRAMES,N_COLS], dtype=tf.float32)\n            ), axis=1)\n        # Downsample\n        data = tf.image.resize(\n            data,\n            [1, N_TARGET_FRAMES],\n            method=tf.image.ResizeMethod.BILINEAR,\n        )\n        \n        # Squeeze Batch Dimension\n        data = tf.squeeze(data, axis=[0])\n        \n        return data\n    \npreprocess_layer = PreprocessLayer()\n\ninputs = group_df[COLUMNS0].values\ninputs = inputs[:1]\n\nframes = preprocess_layer(inputs)\n\nprint(f'inputs shape: {inputs.shape}')\nprint(f'frames shape: {frames.shape}, NaN count: {np.isnan(frames).sum()}')\n\n# 128是的帧个数，164特征维度","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:18:26.08858Z","iopub.execute_input":"2023-06-19T07:18:26.089165Z","iopub.status.idle":"2023-06-19T07:18:26.481641Z","shell.execute_reply.started":"2023-06-19T07:18:26.089114Z","shell.execute_reply":"2023-06-19T07:18:26.479906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 创建数据和标签","metadata":{}},{"cell_type":"code","source":"# Target Arrays Processed Input Videos\n# 创建空的X，存储训练数据\nX = np.zeros([N_SAMPLES, N_TARGET_FRAMES, N_COLS], dtype=np.float32)\n\n# Ordinally Encoded Target With value 59 for pad token\n# 创建空的y，存储数据标签\ny = np.full(shape=[N_SAMPLES, N_TARGET_FRAMES], fill_value=N_UNIQUE_CHARACTERS, dtype=np.int8)\n\n# Phrase Type\n# 创建空的y，存储原始数据标签\ny_phrase_type = np.empty(shape=[N_SAMPLES], dtype=object)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:26:57.087589Z","iopub.execute_input":"2023-06-19T07:26:57.088273Z","iopub.status.idle":"2023-06-19T07:26:57.09936Z","shell.execute_reply.started":"2023-06-19T07:26:57.088222Z","shell.execute_reply":"2023-06-19T07:26:57.097868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# All Unique Parquet Files\n# 每个parpqut文件链接\nUNIQUE_FILE_PATHS = pd.Series(train['file_path'].unique())\nN_UNIQUE_FILE_PATHS = len(UNIQUE_FILE_PATHS)\n\n# Counter to keep track of sample\nrow = 0\ncount = 0\n\n# Compressed Parquet Files\nPath('train_landmark_subsets').mkdir(parents=True, exist_ok=True)\n\n# Numbre Of Frames Per Character\nN_FRAMES_PER_CHARACTER = []\n\n# Minimum Number Of Frames Per Character\nMIN_NUM_FRAMES_PER_CHARACTER = 4\nVALID_IDXS = []\n\n# Fill Arrays\nfor idx, file_path in enumerate(tqdm(UNIQUE_FILE_PATHS)):\n    # Progress Logging\n    print(f'Processed {idx:02d}/{N_UNIQUE_FILE_PATHS} parquet files')\n    \n    # Read parquet file\n    # 读取每个parquet\n    df = pd.read_parquet(file_path)\n    \n    # Save COLUMN Subset of parquet files for TFLite Model verficiation\n    # 保存部分用于验证TFLite Model\n    name = file_path.split('/')[-1]\n    if idx < 10:\n        df[COLUMNS0].to_parquet(f'train_landmark_subsets/{name}', engine='pyarrow', compression='zstd')\n    \n    # Iterate Over Samples\n    # 对于parquet的sequence进行处理\n    for group, group_df in df.groupby('sequence_id'):\n        # Number of Frames Per Character\n        # 如果每个character对应的小于设置的MIN_NUM_FRAMES_PER_CHARACTER，跳过\n        n_frames_per_character =  len(group_df[COLUMNS0].values) / len(train_sequence_id.loc[group, 'phrase_char'])\n        N_FRAMES_PER_CHARACTER.append(n_frames_per_character)\n        if n_frames_per_character < MIN_NUM_FRAMES_PER_CHARACTER:\n            count = count + 1\n            continue\n        \n        else:\n            # Add Valid Index\n            VALID_IDXS.append(count)\n            count = count + 1\n        \n        # Get Processed Frames and non empty frame indices\n        # 处理选择的特征，并进行pad\n        frames = preprocess_layer(group_df[COLUMNS0].values)\n        assert frames.ndim == 2\n        \n        # Assign\n        X[row] = frames\n        \n        # Add Target By Ordinally Encoding Characters\n        # 编码phrase字符\n        phrase_char = train_sequence_id.loc[group, 'phrase_char']\n        for col, char in enumerate(phrase_char):\n            y[row, col] = CHAR2ORD.get(char)\n        \n        # Add EOS Token\n        # 添加结束字符\n        y[row, col+1] = EOS_TOKEN\n        \n        # Phrase Type\n        y_phrase_type[row] = train_sequence_id.loc[group, 'phrase_type']\n        \n        # Row Count\n        row += 1\n    # clean up\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:27:24.262185Z","iopub.execute_input":"2023-06-19T07:27:24.262673Z","iopub.status.idle":"2023-06-19T07:29:09.482228Z","shell.execute_reply.started":"2023-06-19T07:27:24.262636Z","shell.execute_reply":"2023-06-19T07:29:09.480341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rows denotes the number of samples with frames/character above threshold\nprint(f'row: {row}, count: {count}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:29:16.558383Z","iopub.execute_input":"2023-06-19T07:29:16.558811Z","iopub.status.idle":"2023-06-19T07:29:16.565854Z","shell.execute_reply.started":"2023-06-19T07:29:16.558778Z","shell.execute_reply":"2023-06-19T07:29:16.564439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example target, note the phrase is padded with the pad token 59\nprint(f'Example Target: {y[0]}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:30:07.152814Z","iopub.execute_input":"2023-06-19T07:30:07.153914Z","iopub.status.idle":"2023-06-19T07:30:07.162069Z","shell.execute_reply.started":"2023-06-19T07:30:07.153865Z","shell.execute_reply":"2023-06-19T07:30:07.160523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filer X/y\nX = X[:row]\ny = y[:row]","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:30:09.176763Z","iopub.execute_input":"2023-06-19T07:30:09.177723Z","iopub.status.idle":"2023-06-19T07:30:09.183428Z","shell.execute_reply.started":"2023-06-19T07:30:09.177669Z","shell.execute_reply":"2023-06-19T07:30:09.182147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save X/y\n# 保存数据和标签\nnp.save('X.npy', X)\nnp.save('y.npy', y)\n\n# Save Validation\n# 划分验证集，按照participant_id划分\nsplitter = GroupShuffleSplit(test_size=0.10, n_splits=2, random_state=SEED)\nPARTICIPANT_IDS = train['participant_id'].values[VALID_IDXS]\ntrain_idxs, val_idxs = next(splitter.split(X, y, groups=PARTICIPANT_IDS))\n\n# Save Train\nnp.save('X_train.npy', X[train_idxs])\nnp.save('y_train.npy', y[train_idxs])\n# Save Validation\nnp.save('X_val.npy', X[val_idxs])\nnp.save('y_val.npy', y[val_idxs])\n# Verify Train/Val is correctly split by participan id\nprint(f'Patient ID Intersection Train/Val: {set(PARTICIPANT_IDS[train_idxs]).intersection(PARTICIPANT_IDS[val_idxs])}')\n# Train/Val Sizes\nprint(f'# Train Samples: {len(train_idxs)}, # Val Samples: {len(val_idxs)}')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:30:34.941768Z","iopub.execute_input":"2023-06-19T07:30:34.942728Z","iopub.status.idle":"2023-06-19T07:30:35.727749Z","shell.execute_reply.started":"2023-06-19T07:30:34.942688Z","shell.execute_reply":"2023-06-19T07:30:35.726327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 统计每个chararcter对应的frame","metadata":{}},{"cell_type":"code","source":"N_FRAMES_PER_CHARACTER_S = pd.Series(N_FRAMES_PER_CHARACTER)\n\ndisplay(N_FRAMES_PER_CHARACTER_S.describe(percentiles=PERCENTILES).to_frame('Value').round(2))\n\nplt.figure(figsize=(20,10))\nplt.title('Number Of Frames Per Phrase Character')\nN_FRAMES_PER_CHARACTER_S.plot(kind='hist', bins=128)\n# Plot till 99th percentile\np99 = math.ceil(np.percentile(N_FRAMES_PER_CHARACTER_S, 99))\nplt.xticks(np.arange(0, p99+1, 1))\nplt.xlim(0, p99)\nplt.xlabel('Number Of Frames Per Phrase Character')\nplt.ylabel('Sample Count')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:30:55.727225Z","iopub.execute_input":"2023-06-19T07:30:55.727672Z","iopub.status.idle":"2023-06-19T07:30:56.504303Z","shell.execute_reply.started":"2023-06-19T07:30:55.727638Z","shell.execute_reply":"2023-06-19T07:30:56.502836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 统计选择特征的均值和方差","metadata":{}},{"cell_type":"code","source":"def get_left_right_hand_mean_std():\n    # Dominant Hand Statistics\n    MEANS = np.zeros([N_COLS], dtype=np.float32)\n    STDS = np.zeros([N_COLS], dtype=np.float32)\n    \n    # Plot\n    fig, axes = plt.subplots(3, figsize=(20, 3*8))\n    \n    # Iterate over all landmarks\n    for col, v in enumerate(tqdm(X.reshape([-1, N_COLS]).T)):\n        v = v[np.nonzero(v)]\n        # Remove zero values as they are NaN values\n        MEANS[col] = v.astype(np.float32).mean()\n        STDS[col] = v.astype(np.float32).std()\n        if col in LEFT_HAND_IDXS:\n            axes[0].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n        elif col in RIGHT_HAND_IDXS:\n            axes[1].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n        else:\n            axes[2].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n        \n    for ax, name in zip(axes, ['Left Hand', 'Right Hand', 'Lips']):\n        ax.set_title(f'{name}', size=24)\n        ax.tick_params(axis='x', labelsize=8, rotation=45)\n        ax.set_ylim(0.0, 1.0)\n        ax.grid(axis='y')\n\n    plt.show()\n    \n    return MEANS, STDS\n\n# Get Dominant Hand Mean/Standard Deviation\nMEANS, STDS = get_left_right_hand_mean_std()\n# Save Mean/STD to normalize input in neural network model\nnp.save('MEANS.npy', MEANS)\nnp.save('STDS.npy', STDS)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T07:31:04.899557Z","iopub.execute_input":"2023-06-19T07:31:04.900012Z"},"trusted":true},"execution_count":null,"outputs":[]}]}