{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Preprocess and save in tfrecords\nThe following code preprocesses the data and save them into a tfrecord file for fast training.\n\n## Parameters\n* Reference landmark\n* Maximum sequence (frame) length\n* Landmarks to be used\n* Axes to be used (x, y, and/or z)\n* Preprocess method (so far only `norm`)\n\n## Preprocess (the `norm` option)\n1. Get the axes of the selected landmarks\n2. Normalize based on the reference landmark\n3. Pad to maximum sequence length if necessary\n4. Convert `nan` to 0\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-09T05:30:12.399631Z","iopub.execute_input":"2023-08-09T05:30:12.400061Z","iopub.status.idle":"2023-08-09T05:30:12.432684Z","shell.execute_reply.started":"2023-08-09T05:30:12.400006Z","shell.execute_reply":"2023-08-09T05:30:12.431576Z"}}},{"cell_type":"code","source":"import os\nfrom os import path\nimport json\nimport pprint\nimport tensorflow as tf\n# import matplotlib.pyplot as plt\nimport pandas as pd\nimport re\nimport glob\nimport tqdm\nimport shutil\n","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:31.763719Z","iopub.execute_input":"2023-08-10T09:48:31.764195Z","iopub.status.idle":"2023-08-10T09:48:31.772326Z","shell.execute_reply.started":"2023-08-10T09:48:31.764148Z","shell.execute_reply":"2023-08-10T09:48:31.770805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_LMKS = 543\n","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:31.891918Z","iopub.execute_input":"2023-08-10T09:48:31.892328Z","iopub.status.idle":"2023-08-10T09:48:31.897122Z","shell.execute_reply.started":"2023-08-10T09:48:31.892288Z","shell.execute_reply":"2023-08-10T09:48:31.895997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to create a tf.train.Feature for an integer value\ndef _int_feature(value_list):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=value_list))\n\n# Function to create a tf.train.Feature for a float value\ndef _float_feature(value_list):\n    return tf.train.Feature(float_list=tf.train.FloatList(value=value_list))\n\n# Function to create a tf.train.Feature for a byte string value\ndef _bytes_feature(value_list):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=value_list))","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:31.908287Z","iopub.execute_input":"2023-08-10T09:48:31.908634Z","iopub.status.idle":"2023-08-10T09:48:31.914363Z","shell.execute_reply.started":"2023-08-10T09:48:31.908605Z","shell.execute_reply":"2023-08-10T09:48:31.913424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_selected_column_idx(all_features, selected_lmks, axes):\n    selected_features = []\n    for lmk in selected_lmks:\n        for axis in axes:\n            selected_features.append(f'{axis}_{lmk}')\n    selected_column_idx = [i for i in range(len(all_features)) if \n                        all_features[i] in selected_features]\n    return selected_column_idx\n\ndef get_refs(inputs, ref_lmk_id):\n    # assume inputs is numpy array of shape [1, seq_len, NUM_LMKS*3] or [seq_len, NUM_LMKS*3]\n    if tf.rank(inputs) == 2:\n        inputs = inputs[None, ...]\n    ref_x = inputs[:, :, [ref_lmk_id]]\n    ref_x = tf.where(tf.math.is_nan(ref_x), tf.zeros_like(ref_x), ref_x)\n    ref_y = inputs[:, :, [ref_lmk_id+NUM_LMKS]]\n    ref_y = tf.where(tf.math.is_nan(ref_y), tf.zeros_like(ref_y), ref_y)\n    ref_z = inputs[:, :, [ref_lmk_id+NUM_LMKS*2]]\n    ref_z = tf.where(tf.math.is_nan(ref_z), tf.zeros_like(ref_z), ref_z)\n    refs = [ref_x, ref_y, ref_z]\n    # nan in refs are filled with 0.0\n    return refs\n\ndef nan_mean(x, axis=None):\n    valid_mask = tf.math.logical_not(tf.math.is_nan(x))\n    sum = tf.math.reduce_sum(tf.where(valid_mask, x, tf.zeros_like(x)), \n                             axis=axis, keepdims=True)\n    weight = tf.math.reduce_sum(tf.cast(valid_mask, tf.float32),\n                                axis=axis, keepdims=True)\n    return sum / weight","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:32.042654Z","iopub.execute_input":"2023-08-10T09:48:32.04308Z","iopub.status.idle":"2023-08-10T09:48:32.05352Z","shell.execute_reply.started":"2023-08-10T09:48:32.043041Z","shell.execute_reply":"2023-08-10T09:48:32.052279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def norm(coord, ref_coord):\n    diff = coord - ref_coord\n    mean_diff = tf.math.sqrt(nan_mean(tf.math.square(diff), axis=-1))\n    norm = diff / mean_diff\n    return norm\n\ndef normalize(coord, refs, axes):\n    axes2num = {'x': 0, 'y': 1, 'z': 2}\n    dim = coord.shape[-1]\n    num_axes = len(axes)\n    step = dim // num_axes\n\n    axis_num = [axes2num[ax] for ax in axes]\n    ref_coord = [refs[i] for i in axis_num]\n    \n    normed_coords = []\n    for k in range(num_axes):\n        # get k-th axis \n        normed_coord = coord[:,:,k*step:(k+1)*step] \n        normed_coord = norm(normed_coord, ref_coord[k])\n        normed_coords.append(normed_coord)\n    normed_coord = tf.concat(normed_coords, axis=-1)\n\n    return normed_coord\n\ndef remove_rows_with_all_nan(coord):\n    # mask for time steps with at least one non-nan value (dim = 1)\n    mask = tf.math.logical_not(tf.reduce_all(tf.math.is_nan(coord), axis=-1))[0]\n    coord = tf.boolean_mask(coord, mask, axis=1)\n    return coord\n        \ndef pad(coord, max_len):\n    seq_len = tf.shape(coord)[1]\n    if seq_len > max_len:\n        coord = coord[:, :max_len, :]\n    else:\n        pad_len = max_len - seq_len\n        coord = tf.pad(coord, [[0,0], [0, pad_len], [0,0]])\n    return coord\n\ndef fill_na(coord):\n    \n    coord = tf.where(tf.math.is_nan(coord), 0.0, coord)\n    return coord ","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:32.17253Z","iopub.execute_input":"2023-08-10T09:48:32.17337Z","iopub.status.idle":"2023-08-10T09:48:32.184502Z","shell.execute_reply.started":"2023-08-10T09:48:32.173327Z","shell.execute_reply":"2023-08-10T09:48:32.183194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preproc_norm (inputs, **kwargs):\n    # assume inputs is numpy array of shape [1, seq_len, NUM_LMKS*3] or [seq_len, NUM_LMKS*3]\n    if tf.rank(inputs) == 2:\n        inputs = inputs[None, ...]\n\n    ref_lmrk_id = kwargs.get('ref_lmrk_id', 0)\n    axes = kwargs.get('axes', ['x', 'y'])\n    selected_lmks = kwargs.get('selected_lmks', [])\n    all_features = kwargs.get('all_features', [])\n    max_len = kwargs.get('max_len', 300)\n\n    refs = get_refs(inputs, ref_lmrk_id)\n    selected_column_idx = get_selected_column_idx(all_features, selected_lmks, axes)\n    coord = tf.gather(inputs, selected_column_idx, axis=-1)\n    coord = remove_rows_with_all_nan(coord)\n    normed_coord = normalize(coord, refs, axes)\n    padded_coord = pad(normed_coord, max_len)\n    output = tf.where(tf.math.is_nan(padded_coord), 0.0, padded_coord)\n\n    return output","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:32.238856Z","iopub.execute_input":"2023-08-10T09:48:32.239584Z","iopub.status.idle":"2023-08-10T09:48:32.255503Z","shell.execute_reply.started":"2023-08-10T09:48:32.239546Z","shell.execute_reply":"2023-08-10T09:48:32.254398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#def preproc_hub(inputs, **kwargs):\n   # preproc_method = kwargs.get('preproc_method', 'norm')\n    #if preproc_method == 'norm':\n       # output = preproc_norm(inputs, **kwargs)\n   # return output\n","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:32.25743Z","iopub.execute_input":"2023-08-10T09:48:32.257738Z","iopub.status.idle":"2023-08-10T09:48:32.270286Z","shell.execute_reply.started":"2023-08-10T09:48:32.257712Z","shell.execute_reply":"2023-08-10T09:48:32.269191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preproc_hub(inputs, **kwargs):\n    preproc_method = kwargs.get('preproc_method', 'norm')\n    if preproc_method == 'norm':\n        output = preproc_norm(inputs, **kwargs)\n    return output\ndef create_tf_example(df, df_label, **kwargs):\n    sequence_id = df.index[0]\n    label = df_label[df_label['sequence_id'] == sequence_id]['phrase'].values[0]\n    pattern = r\"(x_|y_|z_)\"\n    coords = [col for col in df.columns if re.search(pattern, col) ]\n    data = df[coords].values\n    \n    preproc_data = preproc_hub(data, **kwargs)\n    \n    seq_len = preproc_data.shape[1]\n\n    lm_dim = len(preproc_data)\n    \n    flat_data  = data.flatten()\n\n    feature_dict = {'sequence_id': _int_feature([sequence_id]),\n                    'seq_len': _int_feature([seq_len]),\n                    'lm_dim': _int_feature([lm_dim]),\n                    'data': _float_feature(flat_data),\n                    'label': _bytes_feature([label.encode()])}\n\n    example = tf.train.Example(features=tf.train.Features(feature=feature_dict))\n    return example\n","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:32.271622Z","iopub.execute_input":"2023-08-10T09:48:32.272551Z","iopub.status.idle":"2023-08-10T09:48:32.284626Z","shell.execute_reply.started":"2023-08-10T09:48:32.272516Z","shell.execute_reply":"2023-08-10T09:48:32.283416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def write_to_tfrecord(prquet_file, df_label, writer, **kwargs):\n    df = pd.read_parquet(prquet_file)\n    sequence_ids = df.index.unique()\n    for sequence_id in sequence_ids:\n        sub_df = df.loc[[sequence_id]]\n        example = create_tf_example(sub_df, df_label, **kwargs)\n        writer.write(example.SerializeToString())","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:32.338599Z","iopub.execute_input":"2023-08-10T09:48:32.339024Z","iopub.status.idle":"2023-08-10T09:48:32.345761Z","shell.execute_reply.started":"2023-08-10T09:48:32.33899Z","shell.execute_reply":"2023-08-10T09:48:32.34455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:32.348559Z","iopub.execute_input":"2023-08-10T09:48:32.349048Z","iopub.status.idle":"2023-08-10T09:48:33.958397Z","shell.execute_reply.started":"2023-08-10T09:48:32.349004Z","shell.execute_reply":"2023-08-10T09:48:33.957107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n","metadata":{}},{"cell_type":"markdown","source":"# Main part","metadata":{}},{"cell_type":"markdown","source":"You can edit the parameters here. \nI set `num_files = 10` due to storage capacity, but you can select all if you have enough storage.","metadata":{}},{"cell_type":"code","source":"# Main function\ndata_dir = '/kaggle/input/asl-fingerspelling'\n\ntfrecords_dir = '/kaggle/working/tfrecords'\n\nif os.path.exists(tfrecords_dir):\n    shutil.rmtree(tfrecords_dir)\nos.makedirs(tfrecords_dir, exist_ok=True)\n\ntfrecords_path = path.join(tfrecords_dir, 'train.tfrecords')\npreproc_args_path = path.join(tfrecords_dir, 'preproc_args.json')\n\nparquet_dir = path.join(data_dir, '/kaggle/input/asl-fingerspelling/train_landmarks')\nparquet_paths = glob.glob(path.join(parquet_dir, '*.parquet'))\nlabel_path = path.join(data_dir, '/kaggle/input/asl-fingerspelling/train.csv')\n\ndf_label = pd.read_csv(label_path)\n\nn_hand_lmrks = 21\nn_pose_lmrks = 33\nhandedness = ['left', 'right']\nselected_lmks = []\nfor hand in handedness:\n    for i in range(n_hand_lmrks):\n        selected_lmks.append(f'{hand}_hand_{i}')\nfor i in range(n_pose_lmrks):\n    selected_lmks.append(f'pose_{i}')\n\n# features_path = 'all_features.json'\n# with open(features_path, 'r') as f:\n#     all_features = json.load(f)['all_features']\nsample_df = pd.read_parquet(parquet_paths[0])\nall_features = sample_df.columns[1:]\nprint(all_features)\naxes = ['x', 'y']\n# selected_column_idx = get_selected_column_idx(all_features, selected_lmks, coords)\n\nmax_len = 300\nref_lmrk_id = 489  # x_right_hand_0\npreproc_method = 'norm'\nnum_files = 10\n\npreproc_args = {'max_len': max_len,\n                'ref_lmrk_id': ref_lmrk_id,\n                'preproc_method': preproc_method,\n                'selected_lmks': selected_lmks,\n                'axes': axes, \n                'all_features': all_features}\n\n\nwith tf.io.TFRecordWriter(tfrecords_path) as writer:\n    for parquet_path in tqdm.tqdm(parquet_paths[:num_files]):\n        write_to_tfrecord(parquet_path, df_label, writer, **preproc_args)","metadata":{"execution":{"iopub.status.busy":"2023-08-10T09:48:53.882991Z","iopub.execute_input":"2023-08-10T09:48:53.883391Z","iopub.status.idle":"2023-08-10T10:05:25.025778Z","shell.execute_reply.started":"2023-08-10T09:48:53.88336Z","shell.execute_reply":"2023-08-10T10:05:25.024375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}