{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport os\nimport json\nimport math\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom joblib import Parallel, delayed\nimport tensorflow as tf\nfrom multiprocessing import cpu_count\ncpu_count()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-01T18:33:50.870107Z","iopub.execute_input":"2023-08-01T18:33:50.870527Z","iopub.status.idle":"2023-08-01T18:33:50.879601Z","shell.execute_reply.started":"2023-08-01T18:33:50.870497Z","shell.execute_reply":"2023-08-01T18:33:50.878333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\ndf = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n\nLIP = [\n    61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nREYE = [\n    33, 7, 163, 144, 145, 153, 154, 155, 133,\n    246, 161, 160, 159, 158, 157, 173,\n]\nLEYE = [\n    263, 249, 390, 373, 374, 380, 381, 382, 362,\n    466, 388, 387, 386, 385, 384, 398,\n]\nEYE = REYE + LEYE\n\nNOSE=[\n    1,2,98,327\n]\nLNOSE = [98]\nRNOSE = [327]\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE] + [f'x_face_{i}' for i in LIP] + [f'x_face_{i}' for i in NOSE] + [f'x_face_{i}' for i in EYE]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE] + [f'y_face_{i}' for i in LIP] + [f'y_face_{i}' for i in NOSE] + [f'y_face_{i}' for i in EYE]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE] + [f'z_face_{i}' for i in LIP] + [f'z_face_{i}' for i in NOSE] + [f'z_face_{i}' for i in EYE]\n\n\ndef check(col, arr):\n    num = int(col.split(\"_\")[-1])\n    for i in arr:\n        if i == num:\n            return 1\n    return 0\n\nSEL_COLS = X + Y + Z\n\nLIP_IDX_X   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"x\" in col and check(col, LIP)]\nNOSE_IDX_X   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"x\" in col and check(col, NOSE)]\nEYE_IDX_X   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"x\" in col and check(col, EYE)]\n\n\nRHAND_IDX_X = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"x\" in col]\nLHAND_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"x\" in col]\nRPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE and \"x\" in col]\nLPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE and \"x\" in col]\n\nLIP_IDX_Y   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"y\" in col and check(col, LIP)]\nNOSE_IDX_Y   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"y\" in col and check(col, NOSE)]\nEYE_IDX_Y   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"y\" in col and check(col, EYE)]\n\nRHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"y\" in col]\nLHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"y\" in col]\nRPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE and \"y\" in col]\nLPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE and \"y\" in col]\n\nLIP_IDX_Z   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"z\" in col and check(col, LIP)]\nNOSE_IDX_Z   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"z\" in col and check(col, NOSE)]\nEYE_IDX_Z   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"z\" in col and check(col, EYE)]\nRHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"z\" in col]\nLHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"z\" in col]\nRPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE and \"z\" in col]\nLPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE and \"z\" in col]","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:50.881635Z","iopub.execute_input":"2023-08-01T18:33:50.882065Z","iopub.status.idle":"2023-08-01T18:33:50.970036Z","shell.execute_reply.started":"2023-08-01T18:33:50.882029Z","shell.execute_reply":"2023-08-01T18:33:50.968361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=SEL_COLS)\n\nfile_id = df.file_id.iloc[0]\ninpdir = \"/kaggle/input/asl-fingerspelling/train_landmarks\"\npqfile = f\"{inpdir}/{file_id}.parquet\"\nseq_refs = df.loc[df.file_id == file_id]\nseqs = load_relevant_data_subset(pqfile)\n\nseq_id = seq_refs.sequence_id.iloc[1]\nframes = seqs.iloc[seqs.index == seq_id].to_numpy()\nphrase = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:50.972167Z","iopub.execute_input":"2023-08-01T18:33:50.972526Z","iopub.status.idle":"2023-08-01T18:33:51.324819Z","shell.execute_reply.started":"2023-08-01T18:33:50.972498Z","shell.execute_reply":"2023-08-01T18:33:51.324094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:51.326071Z","iopub.execute_input":"2023-08-01T18:33:51.326534Z","iopub.status.idle":"2023-08-01T18:33:51.332315Z","shell.execute_reply.started":"2023-08-01T18:33:51.326504Z","shell.execute_reply":"2023-08-01T18:33:51.331553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(x):\n    lip_x = x[:, LIP_IDX_X]\n    lip_y = x[:, LIP_IDX_Y]\n    lip_z = x[:, LIP_IDX_Z]\n    \n    nose_x = x[:, NOSE_IDX_X]\n    nose_y = x[:, NOSE_IDX_Y]\n    nose_z = x[:, NOSE_IDX_Z]\n    \n    eye_x = x[:, EYE_IDX_X]\n    eye_y = x[:, EYE_IDX_Y]\n    eye_z = x[:, EYE_IDX_Z]\n\n    rhand_x = x[:, RHAND_IDX_X]\n    rhand_y = x[:, RHAND_IDX_Y]\n    rhand_z = x[:, RHAND_IDX_Z]\n    \n    lhand_x = x[:, LHAND_IDX_X]\n    lhand_y = x[:, LHAND_IDX_Y]\n    lhand_z = x[:, LHAND_IDX_Z]\n\n    rpose_x = x[:, RPOSE_IDX_X]\n    rpose_y = x[:, RPOSE_IDX_Y]\n    rpose_z = x[:, RPOSE_IDX_Z]\n    \n    lpose_x = x[:, LPOSE_IDX_X]\n    lpose_y = x[:, LPOSE_IDX_Y]\n    lpose_z = x[:, LPOSE_IDX_Z]\n    \n    \n    rhand = np.stack([rhand_x, rhand_y, rhand_z], axis=-1)\n    rpose = np.stack([rpose_x, rpose_y, rpose_z], axis=-1)\n    \n    lhand = np.stack([lhand_x, lhand_y, lhand_z], axis=-1)\n    lpose = np.stack([lpose_x, lpose_y, lpose_z], axis=-1)\n    \n    lip = np.stack([lip_x, lip_y, lip_z], axis=-1)\n    eye = np.stack([eye_x, eye_y, eye_z], axis=-1)\n    nose = np.stack([nose_x, nose_y, nose_z], axis=-1)\n\n\n    return rhand, lhand, rpose, lpose, lip, eye, nose\n\nrhand, lhand, rpose, lpose, lip, eye, nose = process(frames)\nprint(rhand.shape, lhand.shape, rpose.shape, lpose.shape, lip.shape, eye.shape, nose.shape)","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:51.334334Z","iopub.execute_input":"2023-08-01T18:33:51.334709Z","iopub.status.idle":"2023-08-01T18:33:51.34572Z","shell.execute_reply.started":"2023-08-01T18:33:51.334688Z","shell.execute_reply":"2023-08-01T18:33:51.344603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:51.34721Z","iopub.execute_input":"2023-08-01T18:33:51.347804Z","iopub.status.idle":"2023-08-01T18:33:51.364638Z","shell.execute_reply.started":"2023-08-01T18:33:51.347776Z","shell.execute_reply":"2023-08-01T18:33:51.363661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen(df):\n    for file_id in df.file_id.unique():\n        pqfile = f\"/kaggle/input/asl-fingerspelling/train_landmarks/{file_id}.parquet\"\n        seq_refs = df.loc[df.file_id == file_id]\n        seqs = load_relevant_data_subset(pqfile)\n\n        for seq_id in seq_refs.sequence_id:\n            x = seqs.iloc[seqs.index == seq_id].to_numpy()\n            y = df.loc[df.sequence_id == seq_id].phrase.iloc[0]\n            uid = df.loc[df.sequence_id == seq_id].participant_id.iloc[0]\n            rhand, lhand, rpose, lpose, lip, eye, nose = process(x)\n            yield rhand, lhand, rpose, lpose, lip, eye, nose","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:51.365663Z","iopub.execute_input":"2023-08-01T18:33:51.365908Z","iopub.status.idle":"2023-08-01T18:33:51.371865Z","shell.execute_reply.started":"2023-08-01T18:33:51.365887Z","shell.execute_reply":"2023-08-01T18:33:51.37092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function(jit_compile=True)\ndef pre_process0(x):\n    lip_x = tf.gather(x, LIP_IDX_X, axis=1)\n    lip_y = tf.gather(x, LIP_IDX_Y, axis=1)\n    lip_z = tf.gather(x, LIP_IDX_Z, axis=1)\n    \n    eye_x = tf.gather(x, EYE_IDX_X, axis=1)\n    eye_y = tf.gather(x, EYE_IDX_Y, axis=1)\n    eye_z = tf.gather(x, EYE_IDX_Z, axis=1)\n    \n    nose_x = tf.gather(x, NOSE_IDX_X, axis=1)\n    nose_y = tf.gather(x, NOSE_IDX_Y, axis=1)\n    nose_z = tf.gather(x, NOSE_IDX_Z, axis=1)\n\n    rhand_x = tf.gather(x, RHAND_IDX_X, axis=1)\n    rhand_y = tf.gather(x, RHAND_IDX_Y, axis=1)\n    rhand_z = tf.gather(x, RHAND_IDX_Z, axis=1)\n    \n    lhand_x = tf.gather(x, LHAND_IDX_X, axis=1)\n    lhand_y = tf.gather(x, LHAND_IDX_Y, axis=1)\n    lhand_z = tf.gather(x, LHAND_IDX_Z, axis=1)\n\n    rpose_x = tf.gather(x, RPOSE_IDX_X, axis=1)\n    rpose_y = tf.gather(x, RPOSE_IDX_Y, axis=1)\n    rpose_z = tf.gather(x, RPOSE_IDX_Z, axis=1)\n    \n    lpose_x = tf.gather(x, LPOSE_IDX_X, axis=1)\n    lpose_y = tf.gather(x, LPOSE_IDX_Y, axis=1)\n    lpose_z = tf.gather(x, LPOSE_IDX_Z, axis=1)\n    \n    lip   = tf.concat([lip_x[..., tf.newaxis], lip_y[..., tf.newaxis], lip_z[..., tf.newaxis]], axis=-1)\n    eye   = tf.concat([eye_x[..., tf.newaxis], eye_y[..., tf.newaxis], eye_z[..., tf.newaxis]], axis=-1)\n    nose   = tf.concat([nose_x[..., tf.newaxis], nose_y[..., tf.newaxis], nose_z[..., tf.newaxis]], axis=-1)\n\n    rhand = tf.concat([rhand_x[..., tf.newaxis], rhand_y[..., tf.newaxis], rhand_z[..., tf.newaxis]], axis=-1)\n    lhand = tf.concat([lhand_x[..., tf.newaxis], lhand_y[..., tf.newaxis], lhand_z[..., tf.newaxis]], axis=-1)\n    rpose = tf.concat([rpose_x[..., tf.newaxis], rpose_y[..., tf.newaxis], rpose_z[..., tf.newaxis]], axis=-1)\n    lpose = tf.concat([lpose_x[..., tf.newaxis], lpose_y[..., tf.newaxis], lpose_z[..., tf.newaxis]], axis=-1)\n    \n    hand = tf.concat([rhand, lhand], axis=1)\n    hand = tf.where(tf.math.is_nan(hand), 0.0, hand)\n    mask = tf.math.not_equal(tf.reduce_sum(hand, axis=[1, 2]), 0.0)\n\n    lip = lip[mask]\n    eye = eye[mask]\n    nose = nose[mask]\n\n    rhand = rhand[mask]\n    lhand = lhand[mask]\n    rpose = rpose[mask]\n    lpose = lpose[mask]\n\n    return lip, rhand, lhand, rpose, lpose, eye, nose\n\npre_process0(frames)","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:51.373758Z","iopub.execute_input":"2023-08-01T18:33:51.374179Z","iopub.status.idle":"2023-08-01T18:33:54.262889Z","shell.execute_reply.started":"2023-08-01T18:33:51.374152Z","shell.execute_reply":"2023-08-01T18:33:54.261932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_files = df.file_id.unique().tolist()\nall_files","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:54.264345Z","iopub.execute_input":"2023-08-01T18:33:54.2647Z","iopub.status.idle":"2023-08-01T18:33:54.277537Z","shell.execute_reply.started":"2023-08-01T18:33:54.264667Z","shell.execute_reply":"2023-08-01T18:33:54.276042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks = [all_files[:23], all_files[23:46], all_files[46:]]","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:54.279132Z","iopub.execute_input":"2023-08-01T18:33:54.279529Z","iopub.status.idle":"2023-08-01T18:33:54.289624Z","shell.execute_reply.started":"2023-08-01T18:33:54.279498Z","shell.execute_reply":"2023-08-01T18:33:54.288478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=SEL_COLS)\n\nif not os.path.isdir(\"tfds\"): os.mkdir(\"tfds\")\n\ndef process_chunk(files):\n    for file_id in tqdm(files):\n        pqfile = f\"{inpdir}/{file_id}.parquet\"\n        seq_refs = df.loc[df.file_id == file_id]\n        seqs = load_relevant_data_subset(pqfile)\n\n        for uid, seq_id, phrase in zip(seq_refs.participant_id, seq_refs.sequence_id, seq_refs.phrase):\n            tffile = f\"tfds/{uid}_{seq_id}.tfrecord\"\n            with tf.io.TFRecordWriter(tffile) as file_writer:\n                frames = seqs.iloc[seqs.index == seq_id].to_numpy()\n                lip, rhand, lhand, rpose, lpose, eye, nose = pre_process0(frames)\n\n                features = {}\n                features[\"lip\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(lip, -1).numpy())) \n                features[\"eye\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(eye, -1).numpy())) \n                features[\"nose\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(nose, -1).numpy())) \n                features[\"rhand\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(rhand, -1).numpy())) \n                features[\"lhand\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(lhand, -1).numpy())) \n                features[\"rpose\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(rpose, -1).numpy())) \n                features[\"lpose\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(lpose, -1).numpy())) \n                features[\"phrase\"] = tf.train.Feature(int64_list=tf.train.Int64List(value=[char_to_num[x] for x in phrase]))\n\n                record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n                file_writer.write(record_bytes)","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:54.293195Z","iopub.execute_input":"2023-08-01T18:33:54.293855Z","iopub.status.idle":"2023-08-01T18:33:54.309216Z","shell.execute_reply.started":"2023-08-01T18:33:54.293827Z","shell.execute_reply":"2023-08-01T18:33:54.307997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = Parallel(n_jobs=cpu_count(), prefer=\"threads\")(\n    delayed(process_chunk)(x)\n    for x in chunks\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:33:54.311551Z","iopub.execute_input":"2023-08-01T18:33:54.312026Z","iopub.status.idle":"2023-08-01T18:35:56.157697Z","shell.execute_reply.started":"2023-08-01T18:33:54.311965Z","shell.execute_reply":"2023-08-01T18:35:56.154699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_fn(record_bytes):\n    schema = {\n        \"lip\": tf.io.VarLenFeature(tf.float32),\n        \"eye\": tf.io.VarLenFeature(tf.float32),\n        \"nose\": tf.io.VarLenFeature(tf.float32),\n        \"rhand\": tf.io.VarLenFeature(tf.float32),\n        \"lhand\": tf.io.VarLenFeature(tf.float32),\n        \"rpose\": tf.io.VarLenFeature(tf.float32),\n        \"lpose\": tf.io.VarLenFeature(tf.float32),\n        \"phrase\": tf.io.VarLenFeature(tf.int64)\n    }\n    x = tf.io.parse_single_example(record_bytes, schema)\n\n    lip = tf.reshape(tf.sparse.to_dense(x[\"lip\"]), (-1, 40, 3))\n    eye = tf.reshape(tf.sparse.to_dense(x[\"eye\"]), (-1, 32, 3))\n    nose = tf.reshape(tf.sparse.to_dense(x[\"nose\"]), (-1, 4, 3))\n    rhand = tf.reshape(tf.sparse.to_dense(x[\"rhand\"]), (-1, 21, 3))\n    lhand = tf.reshape(tf.sparse.to_dense(x[\"lhand\"]), (-1, 21, 3))\n    rpose = tf.reshape(tf.sparse.to_dense(x[\"rpose\"]), (-1, 5, 3))\n    lpose = tf.reshape(tf.sparse.to_dense(x[\"lpose\"]), (-1, 5, 3))\n    phrase = tf.sparse.to_dense(x[\"phrase\"])\n\n    return lip, rhand, lhand, rpose, lpose, eye, nose, phrase\n\n    \ntffiles = [f\"tfds/{file_id}.tfrecord\" for file_id in df.file_id.unique()]\nfor batch in tf.data.TFRecordDataset(tffiles).map(decode_fn).take(1):\n    print(batch[0].shape, batch[1].shape, batch[2].shape, batch[3].shape, batch[4].shape, batch[5])","metadata":{"execution":{"iopub.status.busy":"2023-08-01T18:35:56.160937Z","iopub.status.idle":"2023-08-01T18:35:56.161475Z","shell.execute_reply.started":"2023-08-01T18:35:56.161236Z","shell.execute_reply":"2023-08-01T18:35:56.161257Z"},"trusted":true},"execution_count":null,"outputs":[]}]}