{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport os\nimport json\nimport math\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\ndf = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n\nLIP = [\n    61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE] + [f'x_face_{i}' for i in LIP]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE] + [f'y_face_{i}' for i in LIP]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE] + [f'z_face_{i}' for i in LIP]\n\nSEL_COLS = X + Y + Z\n\nLIP_IDX_X   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"x\" in col]\nRHAND_IDX_X = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"x\" in col]\nLHAND_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"x\" in col]\nRPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE and \"x\" in col]\nLPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE and \"x\" in col]\n\nLIP_IDX_Y   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"y\" in col]\nRHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"y\" in col]\nLHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"y\" in col]\nRPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE and \"y\" in col]\nLPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE and \"y\" in col]\n\nLIP_IDX_Z   = [i for i, col in enumerate(SEL_COLS)  if  \"face\" in col and \"z\" in col]\nRHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if \"right\" in col and \"z\" in col]\nLHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"left\" in col and \"z\" in col]\nRPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in RPOSE and \"z\" in col]\nLPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS)  if  \"pose\" in col and int(col[-2:]) in LPOSE and \"z\" in col]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=SEL_COLS)\n\nfile_id = df.file_id.iloc[0]\ninpdir = \"/kaggle/input/asl-fingerspelling/train_landmarks\"\npqfile = f\"{inpdir}/{file_id}.parquet\"\nseq_refs = df.loc[df.file_id == file_id]\nseqs = load_relevant_data_subset(pqfile)\n\nseq_id = seq_refs.sequence_id.iloc[0]\nframes = seqs.iloc[seqs.index == seq_id].to_numpy()\nphrase = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(x):\n    lip_x = x[:, LIP_IDX_X]\n    lip_y = x[:, LIP_IDX_Y]\n    lip_z = x[:, LIP_IDX_Z]\n\n    rhand_x = x[:, RHAND_IDX_X]\n    rhand_y = x[:, RHAND_IDX_Y]\n    rhand_z = x[:, RHAND_IDX_Z]\n    \n    lhand_x = x[:, LHAND_IDX_X]\n    lhand_y = x[:, LHAND_IDX_Y]\n    lhand_z = x[:, LHAND_IDX_Z]\n\n    rpose_x = x[:, RPOSE_IDX_X]\n    rpose_y = x[:, RPOSE_IDX_Y]\n    rpose_z = x[:, RPOSE_IDX_Z]\n    \n    lpose_x = x[:, LPOSE_IDX_X]\n    lpose_y = x[:, LPOSE_IDX_Y]\n    lpose_z = x[:, LPOSE_IDX_Z]\n    \n    rhnonans = ~np.isnan(np.sum(rhand_x, axis=1))\n    lhnonans = ~np.isnan(np.sum(lhand_x, axis=1))\n    lpnonans = ~np.isnan(np.sum(lip_x,   axis=1))\n    \n    rhand = np.stack([rhand_x, rhand_y, rhand_z], axis=-1)[rhnonans]\n    rpose = np.stack([rpose_x, rpose_y, rpose_z], axis=-1)[rhnonans]\n    \n    lhand = np.stack([lhand_x, lhand_y, lhand_z], axis=-1)[lhnonans]\n    lpose = np.stack([lpose_x, lpose_y, lpose_z], axis=-1)[lhnonans]\n    \n    lip = np.stack([lip_x, lip_y, lip_z], axis=-1)[lpnonans]\n\n    return rhand, lhand, rpose, lpose, lip\n\nrhand, lhand, rpose, lpose, lip = process(frames)\nprint(rhand.shape, lhand.shape, rpose.shape, lpose.shape, lip.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen(df):\n    for file_id in df.file_id.unique():\n        pqfile = f\"/kaggle/input/asl-fingerspelling/train_landmarks/{file_id}.parquet\"\n        seq_refs = df.loc[df.file_id == file_id]\n        seqs = load_relevant_data_subset(pqfile)\n\n        for seq_id in seq_refs.sequence_id:\n            x = seqs.iloc[seqs.index == seq_id].to_numpy()\n            y = df.loc[df.sequence_id == seq_id].phrase.iloc[0]\n            rhand, lhand, rpose, lpose, lip = process(x)\n            \n            if max(rhand.shape[0], lhand.shape[0]) > len(y):\n                yield rhand, lhand, rpose, lpose, lip","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RHAND = []\nLHAND = []\nRPOSE = []\nLPOSE = []\nLIP = []\n\nfor rhand, lhand, rpose, lpose, lip in tqdm(gen(df)):\n    RHAND.extend(rhand)\n    LHAND.extend(lhand)\n    RPOSE.extend(rpose)\n    LPOSE.extend(lpose)\n    LIP.extend(lip)\n    \nRHAND = np.array(RHAND)\nLHAND = np.array(LHAND)\nRPOSE = np.array(RPOSE)\nLPOSE = np.array(LPOSE)\nLIP = np.array(LIP)\ngc.collect()\n\nrh_mean = np.mean(RHAND, axis=0)\nlh_mean = np.mean(LHAND, axis=0)\nrp_mean = np.mean(RPOSE, axis=0)\nlp_mean = np.mean(LPOSE, axis=0)\nlip_mean = np.mean(LIP, axis=0)\n\nrh_std = np.std(RHAND, axis=0)\nlh_std = np.std(LHAND, axis=0)\nrp_std = np.std(RPOSE, axis=0)\nlp_std = np.std(LPOSE, axis=0)\nlip_std = np.std(LIP, axis=0)\n\n!mkdir mean_std\nnp.save(\"mean_std/rh_mean.npy\", rh_mean)\nnp.save(\"mean_std/lh_mean.npy\", lh_mean)\nnp.save(\"mean_std/rp_mean.npy\", rp_mean)\nnp.save(\"mean_std/lp_mean.npy\", lp_mean)\nnp.save(\"mean_std/lip_mean.npy\", lip_mean)\n\nnp.save(\"mean_std/rh_std.npy\", rh_std)\nnp.save(\"mean_std/lh_std.npy\", lh_std)\nnp.save(\"mean_std/rp_std.npy\", rp_std)\nnp.save(\"mean_std/lp_std.npy\", lp_std)\nnp.save(\"mean_std/lip_std.npy\", lip_std)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function(jit_compile=True)\ndef pre_process0(x):\n    lip_x = tf.gather(x, LIP_IDX_X, axis=1)\n    lip_y = tf.gather(x, LIP_IDX_Y, axis=1)\n    lip_z = tf.gather(x, LIP_IDX_Z, axis=1)\n\n    rhand_x = tf.gather(x, RHAND_IDX_X, axis=1)\n    rhand_y = tf.gather(x, RHAND_IDX_Y, axis=1)\n    rhand_z = tf.gather(x, RHAND_IDX_Z, axis=1)\n    \n    lhand_x = tf.gather(x, LHAND_IDX_X, axis=1)\n    lhand_y = tf.gather(x, LHAND_IDX_Y, axis=1)\n    lhand_z = tf.gather(x, LHAND_IDX_Z, axis=1)\n\n    rpose_x = tf.gather(x, RPOSE_IDX_X, axis=1)\n    rpose_y = tf.gather(x, RPOSE_IDX_Y, axis=1)\n    rpose_z = tf.gather(x, RPOSE_IDX_Z, axis=1)\n    \n    lpose_x = tf.gather(x, LPOSE_IDX_X, axis=1)\n    lpose_y = tf.gather(x, LPOSE_IDX_Y, axis=1)\n    lpose_z = tf.gather(x, LPOSE_IDX_Z, axis=1)\n    \n    lip   = tf.concat([lip_x[..., tf.newaxis], lip_y[..., tf.newaxis], lip_z[..., tf.newaxis]], axis=-1)\n    rhand = tf.concat([rhand_x[..., tf.newaxis], rhand_y[..., tf.newaxis], rhand_z[..., tf.newaxis]], axis=-1)\n    lhand = tf.concat([lhand_x[..., tf.newaxis], lhand_y[..., tf.newaxis], lhand_z[..., tf.newaxis]], axis=-1)\n    rpose = tf.concat([rpose_x[..., tf.newaxis], rpose_y[..., tf.newaxis], rpose_z[..., tf.newaxis]], axis=-1)\n    lpose = tf.concat([lpose_x[..., tf.newaxis], lpose_y[..., tf.newaxis], lpose_z[..., tf.newaxis]], axis=-1)\n    \n    hand = tf.concat([rhand, lhand], axis=1)\n    hand = tf.where(tf.math.is_nan(hand), 0.0, hand)\n    mask = tf.math.not_equal(tf.reduce_sum(hand, axis=[1, 2]), 0.0)\n\n    lip = lip[mask]\n    rhand = rhand[mask]\n    lhand = lhand[mask]\n    rpose = rpose[mask]\n    lpose = lpose[mask]\n\n    return lip, rhand, lhand, rpose, lpose\n\npre_process0(frames)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=SEL_COLS)\n\nif not os.path.isdir(\"tfds\"): os.mkdir(\"tfds\")\n\nfor file_id in tqdm(df.file_id.unique()):\n    pqfile = f\"{inpdir}/{file_id}.parquet\"\n    tffile = f\"tfds/{file_id}.tfrecord\"\n    seq_refs = df.loc[df.file_id == file_id]\n    seqs = load_relevant_data_subset(pqfile)\n    \n    with tf.io.TFRecordWriter(tffile) as file_writer:\n        for seq_id, phrase in zip(seq_refs.sequence_id, seq_refs.phrase):\n            frames = seqs.iloc[seqs.index == seq_id].to_numpy()\n            \n            lip, rhand, lhand, rpose, lpose = pre_process0(frames)\n\n            if max(rhand.shape[0], lhand.shape[0]) < len(phrase):\n                continue\n                \n            features = {}\n            features[\"lip\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(lip, -1).numpy())) \n            features[\"rhand\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(rhand, -1).numpy())) \n            features[\"lhand\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(lhand, -1).numpy())) \n            features[\"rpose\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(rpose, -1).numpy())) \n            features[\"lpose\"] = tf.train.Feature(float_list=tf.train.FloatList(value=tf.reshape(lpose, -1).numpy())) \n            features[\"phrase\"] = tf.train.Feature(int64_list=tf.train.Int64List(value=[char_to_num[x] for x in phrase]))\n            \n            record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n            file_writer.write(record_bytes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_fn(record_bytes):\n    schema = {\n        \"lip\": tf.io.VarLenFeature(tf.float32),\n        \"rhand\": tf.io.VarLenFeature(tf.float32),\n        \"lhand\": tf.io.VarLenFeature(tf.float32),\n        \"rpose\": tf.io.VarLenFeature(tf.float32),\n        \"lpose\": tf.io.VarLenFeature(tf.float32),\n        \"phrase\": tf.io.VarLenFeature(tf.int64)\n    }\n    x = tf.io.parse_single_example(record_bytes, schema)\n\n    lip = tf.reshape(tf.sparse.to_dense(x[\"lip\"]), (-1, 40, 3))\n    rhand = tf.reshape(tf.sparse.to_dense(x[\"rhand\"]), (-1, 21, 3))\n    lhand = tf.reshape(tf.sparse.to_dense(x[\"lhand\"]), (-1, 21, 3))\n    rpose = tf.reshape(tf.sparse.to_dense(x[\"rpose\"]), (-1, 5, 3))\n    lpose = tf.reshape(tf.sparse.to_dense(x[\"lpose\"]), (-1, 5, 3))\n    phrase = tf.sparse.to_dense(x[\"phrase\"])\n\n    return lip, rhand, lhand, rpose, lpose, phrase\n\n    \ntffiles = [f\"tfds/{file_id}.tfrecord\" for file_id in df.file_id.unique()]\nfor batch in tf.data.TFRecordDataset(tffiles).map(decode_fn).take(1):\n    print(batch[0].shape, batch[1].shape, batch[2].shape, batch[3].shape, batch[4].shape, batch[5])","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}