{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\nimport tensorflow as tf\nimport pyarrow.parquet as pq\n\nfrom tqdm.notebook import tqdm\n\nfrom skimage.transform import resize\nfrom sklearn.model_selection import train_test_split\nfrom leven import levenshtein\n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport time\nimport json\n\ntqdm.pandas()\n\nprint(f'Tensorflow Version {tf.__version__}')\nprint(f'Python Version: {sys.version}')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-11T04:36:15.390751Z","iopub.execute_input":"2023-07-11T04:36:15.391137Z","iopub.status.idle":"2023-07-11T04:36:25.670344Z","shell.execute_reply.started":"2023-07-11T04:36:15.391108Z","shell.execute_reply":"2023-07-11T04:36:25.669391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\ntrain_supplemental = pd.read_csv('/kaggle/input/asl-fingerspelling/supplemental_metadata.csv')\nprint(\"Len Train: \", len(train))\nprint(train.head(10))","metadata":{"execution":{"iopub.status.busy":"2023-07-11T04:36:25.67191Z","iopub.execute_input":"2023-07-11T04:36:25.672502Z","iopub.status.idle":"2023-07-11T04:36:25.935908Z","shell.execute_reply.started":"2023-07-11T04:36:25.67248Z","shell.execute_reply":"2023-07-11T04:36:25.934663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_save = \"/kaggle/working\"\nroot_data = \"/kaggle/input/asl-fingerspelling\"\nsample = pd.read_parquet(root_data + \"/train_landmarks/5414471.parquet\")\nprint(\"Shape: \", sample.shape)\ndisplay(sample.head(10))","metadata":{"execution":{"iopub.status.busy":"2023-07-11T04:36:25.938673Z","iopub.execute_input":"2023-07-11T04:36:25.938998Z","iopub.status.idle":"2023-07-11T04:36:46.628523Z","shell.execute_reply.started":"2023-07-11T04:36:25.938972Z","shell.execute_reply":"2023-07-11T04:36:46.627106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LIP = [\n    61, 185, 40, 39, 37, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\n\nNOSE = [1, 2, 98, 327]\n    \nLEYE = [466, 388, 387, 386, 385, 384, 398,\n        263, 249, 390, 373, 374, 380, 381, 382, 362]\n    \nREYE = [246, 161, 160, 159, 158, 157, 173,\n        33, 7, 163, 144, 145, 153, 154, 155, 133]\n\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nLIP = [f'x_face_{i}' for i in LIP] + [f'y_face_{i}' for i in LIP] + [f'z_face_{i}' for i in LIP]\nNOSE = [f'x_face_{i}' for i in NOSE] + [f'y_face_{i}' for i in NOSE] + [f'z_face_{i}' for i in NOSE]\nLEYE = [f'x_face_{i}' for i in LEYE] + [f'y_face_{i}' for i in LEYE] + [f'z_face_{i}' for i in LEYE]\nREYE = [f'x_face_{i}' for i in REYE] + [f'y_face_{i}' for i in REYE] + [f'z_face_{i}' for i in REYE]\n\nLHAND = [f'x_left_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)]\nRHAND = [f'x_right_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)] + [f'z_right_hand_{i}' for i in range(21)]\nPOSE = [f'x_pose_{i}' for i in POSE] + [f'y_pose_{i}' for i in POSE] + [f'z_pose_{i}' for i in POSE]\n\nFEATURE_COL = LIP + NOSE + LEYE + REYE + LHAND + RHAND + POSE\nFRAME_LEN = 128","metadata":{"execution":{"iopub.status.busy":"2023-07-11T04:36:46.630379Z","iopub.execute_input":"2023-07-11T04:36:46.631286Z","iopub.status.idle":"2023-07-11T04:36:46.644791Z","shell.execute_reply.started":"2023-07-11T04:36:46.631234Z","shell.execute_reply":"2023-07-11T04:36:46.643634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_IDX = [i for i, col in enumerate(FEATURE_COL) if \"x_\" in col]\nY_IDX = [i for i, col in enumerate(FEATURE_COL) if \"y_\" in col]\nZ_IDX = [i for i, col in enumerate(FEATURE_COL) if \"z_\" in col]\n\nRHAND_IDX = [i for i, col in enumerate(FEATURE_COL) if \"right\" in col]\nLHAND_IDX = [i for i, col in enumerate(FEATURE_COL) if \"left\" in col]\nRPOSE_IDX = [i for i, col in enumerate(FEATURE_COL) if \"pose\" in col and int(col[-2:]) in RPOSE]\nLPOSE_IDX = [i for i, col in enumerate(FEATURE_COL) if \"pose\" in col and int(col[-2:]) in LPOSE]","metadata":{"execution":{"iopub.status.busy":"2023-07-11T04:36:46.645945Z","iopub.execute_input":"2023-07-11T04:36:46.64625Z","iopub.status.idle":"2023-07-11T04:36:46.664302Z","shell.execute_reply.started":"2023-07-11T04:36:46.646226Z","shell.execute_reply":"2023-07-11T04:36:46.662885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs(\"/kaggle/working/train\", exist_ok=True)\nfor file_id in tqdm(train.file_id.unique()):\n    pq_file = f\"{root_data}/train_landmarks/{file_id}.parquet\"\n    tf_file = f\"{root_save}/train/{file_id}.tfrecord\"\n    file_df = train.loc[train[\"file_id\"] == file_id]\n    \n    # read parquet with just selected features\n    parquet_df = pq.read_table(pq_file, columns=['sequence_id'] + FEATURE_COL).to_pandas()\n    parquet_numpy = parquet_df.to_numpy()\n    \n    options = tf.io.TFRecordOptions(compression_type='GZIP', compression_level=9)\n    with tf.io.TFRecordWriter(tf_file, options=options) as file_writer:\n        for seq_id, phrase in zip(file_df.sequence_id, file_df.phrase):\n            frames = parquet_numpy[parquet_df.index == seq_id]\n            \n            # Calculate the number of NaN values in each hand landmark\n            r_nonan = np.sum(np.sum(np.isnan(frames[:, RHAND_IDX]), axis = 1) == 0)\n            l_nonan = np.sum(np.sum(np.isnan(frames[:, LHAND_IDX]), axis = 1) == 0)\n            no_nan = max(r_nonan, l_nonan)\n            \n            if 2*len(phrase) < no_nan:\n                features = {FEATURE_COL[i]: tf.train.Feature(float_list=tf.train.FloatList(value=frames[:, i])) for i in range(len(FEATURE_COL))}\n                features[\"phrase\"] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[bytes(phrase, 'utf-8')]))\n                record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n                file_writer.write(record_bytes)\n\n        file_writer.close()","metadata":{"execution":{"iopub.status.busy":"2023-07-11T04:36:46.665789Z","iopub.execute_input":"2023-07-11T04:36:46.666721Z","iopub.status.idle":"2023-07-11T04:38:43.698484Z","shell.execute_reply.started":"2023-07-11T04:36:46.666683Z","shell.execute_reply":"2023-07-11T04:38:43.697199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_fn(record_bytes):\n    schema = {COL: tf.io.VarLenFeature(dtype=tf.float32) for COL in FEATURE_COL}\n    schema[\"phrase\"] = tf.io.FixedLenFeature([], dtype=tf.string)\n    features = tf.io.parse_single_example(record_bytes, schema)\n    phrase = features[\"phrase\"]\n    landmarks = ([tf.sparse.to_dense(features[COL]) for COL in FEATURE_COL])\n    # Transpose to maintain the original shape of landmarks data.\n    landmarks = tf.transpose(landmarks)\n    \n    return landmarks, phrase\n\nfor file_id in train.file_id:\n    pqfile = f\"{root_data}/train_landmarks/{file_id}.parquet\"\n    tffile = f\"{root_save}/train/{file_id}.tfrecord\"\n    for batch in tf.data.TFRecordDataset([tffile],num_parallel_reads=tf.data.AUTOTUNE, compression_type='GZIP').map(decode_fn).take(2):\n        print(list(batch)[0].shape, list(batch)[1])\n    break","metadata":{"execution":{"iopub.status.busy":"2023-07-11T04:40:44.025939Z","iopub.execute_input":"2023-07-11T04:40:44.026366Z","iopub.status.idle":"2023-07-11T04:40:44.950477Z","shell.execute_reply.started":"2023-07-11T04:40:44.026337Z","shell.execute_reply":"2023-07-11T04:40:44.948967Z"},"trusted":true},"execution_count":null,"outputs":[]}]}