{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\nimport tensorflow as tf\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit\nfrom pathlib import Path\n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport time\nimport json\nimport re\n\nfrom sklearn.model_selection import StratifiedGroupKFold, KFold\nfrom multiprocessing import cpu_count\nfrom joblib import Parallel, delayed\n\n# TQDM Progress Bar With Pandas Apply Function\ntqdm.pandas()\ncpu_count()","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:31.121154Z","iopub.execute_input":"2023-07-19T10:18:31.121498Z","iopub.status.idle":"2023-07-19T10:18:31.133273Z","shell.execute_reply.started":"2023-07-19T10:18:31.121469Z","shell.execute_reply":"2023-07-19T10:18:31.131648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read Character to Ordinal Encoding Mapping\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    CHAR2ORD = json.load(json_file)\ndisplay(pd.Series(CHAR2ORD).to_frame('Ordinal Encoding'))\nN_UNIQUE_CHARACTERS = len(CHAR2ORD)\nprint(N_UNIQUE_CHARACTERS)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:31.373738Z","iopub.execute_input":"2023-07-19T10:18:31.374093Z","iopub.status.idle":"2023-07-19T10:18:31.402205Z","shell.execute_reply.started":"2023-07-19T10:18:31.374065Z","shell.execute_reply":"2023-07-19T10:18:31.400916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\ntrain_df.head()\nprint(f'#Samples: {len(train_df)}, #Unique phrase: {len(train_df[\"phrase\"].unique())}')","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:32.131779Z","iopub.execute_input":"2023-07-19T10:18:32.132491Z","iopub.status.idle":"2023-07-19T10:18:32.212472Z","shell.execute_reply.started":"2023-07-19T10:18:32.132452Z","shell.execute_reply":"2023-07-19T10:18:32.211451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_file_path(path):\n    return f'/kaggle/input/asl-fingerspelling/{path}'\ntrain_df['file_path'] = train_df['path'].apply(get_file_path)\ntrain_df['phrase_char'] = train_df['phrase'].apply(tuple)\ntrain_df['phrase_char_len'] = train_df['phrase_char'].apply(len)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:34.527804Z","iopub.execute_input":"2023-07-19T10:18:34.528159Z","iopub.status.idle":"2023-07-19T10:18:34.611218Z","shell.execute_reply.started":"2023-07-19T10:18:34.528131Z","shell.execute_reply":"2023-07-19T10:18:34.610428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Selected columns","metadata":{}},{"cell_type":"code","source":"LIP = [ 0, \n    61, 185, 40, 39, 37, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\nLLIP = [84,181,91,146,61,185,40,39,37,87,178,88,95,78,191,80,81,82]\nRLIP = [314,405,321,375,291,409,270,269,267,317,402,318,324,308,415,310,311,312]\n\nNOSE = [1,2,98,327]\nLNOSE = [98]\nRNOSE = [327]\n\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nREYE = [\n    33, 7, 163, 144, 145, 153, 154, 155, 133,\n    246, 161, 160, 159, 158, 157, 173,\n]\nLEYE = [\n    263, 249, 390, 373, 374, 380, 381, 382, 362,\n    466, 388, 387, 386, 385, 384, 398,\n]\nEYE = LEYE + REYE\n\nHAND = np.arange(21)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:41.685918Z","iopub.execute_input":"2023-07-19T10:18:41.686286Z","iopub.status.idle":"2023-07-19T10:18:41.695272Z","shell.execute_reply.started":"2023-07-19T10:18:41.686256Z","shell.execute_reply":"2023-07-19T10:18:41.694318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_names(idxs, keyword):\n    names = []\n    for idx in idxs:\n        names.append(keyword + \"_\" + str(idx))\n    return names\n\ndef get_xy_names(names, axis):\n    output = []\n    for n in names:\n        output.append(axis + \"_\" + n)\n    return output","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:42.358594Z","iopub.execute_input":"2023-07-19T10:18:42.358934Z","iopub.status.idle":"2023-07-19T10:18:42.3652Z","shell.execute_reply.started":"2023-07-19T10:18:42.358908Z","shell.execute_reply":"2023-07-19T10:18:42.363688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LIP_NAMES = get_names(LIP, \"face\")\nLLIP_NAMES = get_names(LLIP, \"face\")\nRLIP_NAMES = get_names(RLIP, \"face\")\n\nNOSE_NAMES = get_names(NOSE, \"face\")\nLNOSE_NAMES = get_names(LNOSE, \"face\")\nRNOSE_NAMES = get_names(RNOSE, \"face\")\n\nEYE_NAMES = get_names(EYE, \"face\")\nLEYE_NAMES = get_names(LEYE, \"face\")\nREYE_NAMES = get_names(REYE, \"face\")\n\n#POSE_NAMES = get_names(POSE, \"pose\")\n#LPOSE_NAMES = get_names(LPOSE, \"pose\")\n#RPOSE_NAMES = get_names(RPOSE, \"pose\")\n\nLHAND_NAMES = get_names(HAND, \"left_hand\")\nRHAND_NAMES = get_names(HAND, \"right_hand\")\nHAND_NAMES = LHAND_NAMES + RHAND_NAMES","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:43.438785Z","iopub.execute_input":"2023-07-19T10:18:43.439172Z","iopub.status.idle":"2023-07-19T10:18:43.445461Z","shell.execute_reply.started":"2023-07-19T10:18:43.439143Z","shell.execute_reply":"2023-07-19T10:18:43.444427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"POINT_LANDMARKS = NOSE_NAMES + LIP_NAMES + EYE_NAMES + LHAND_NAMES + RHAND_NAMES\nX_POINT_LANDMARKS = get_xy_names(POINT_LANDMARKS, axis=\"x\")\nY_POINT_LANDMARKS = get_xy_names(POINT_LANDMARKS, axis=\"y\")\nXY_POINT_LANDMARKS = X_POINT_LANDMARKS + Y_POINT_LANDMARKS\n\nNUM_NODES = len(XY_POINT_LANDMARKS)\nN_DIMS = 2\nRATIO = 4\n\nprint(f'NUM_NODES: {NUM_NODES}, N_DIMS: {N_DIMS}')","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:44.432979Z","iopub.execute_input":"2023-07-19T10:18:44.43355Z","iopub.status.idle":"2023-07-19T10:18:44.440718Z","shell.execute_reply.started":"2023-07-19T10:18:44.43351Z","shell.execute_reply":"2023-07-19T10:18:44.439539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LLIP_IDXS = np.argwhere(np.isin(POINT_LANDMARKS, LLIP_NAMES)).squeeze()\nLNOSE_IDXS = np.argwhere(np.isin(POINT_LANDMARKS, LNOSE_NAMES)).squeeze(axis=0)\nLEYE_IDXS = np.argwhere(np.isin(POINT_LANDMARKS, LEYE_NAMES)).squeeze()\nLHAND_IDXS = np.argwhere(np.isin(POINT_LANDMARKS, LHAND_NAMES)).squeeze()\n\nRLIP_IDXS = np.argwhere(np.isin(POINT_LANDMARKS, RLIP_NAMES)).squeeze()\nRNOSE_IDXS = np.argwhere(np.isin(POINT_LANDMARKS, RNOSE_NAMES)).squeeze(axis=0)\nREYE_IDXS = np.argwhere(np.isin(POINT_LANDMARKS, REYE_NAMES)).squeeze()\nRHAND_IDXS = np.argwhere(np.isin(POINT_LANDMARKS, RHAND_NAMES)).squeeze()","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:45.373769Z","iopub.execute_input":"2023-07-19T10:18:45.37412Z","iopub.status.idle":"2023-07-19T10:18:45.384555Z","shell.execute_reply.started":"2023-07-19T10:18:45.374092Z","shell.execute_reply":"2023-07-19T10:18:45.383246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LHAND_X_NAMES =  get_xy_names(LHAND_NAMES, axis=\"x\")\nLHAND_Y_NAMES =  get_xy_names(LHAND_NAMES, axis=\"y\")\nRHAND_X_NAMES =  get_xy_names(RHAND_NAMES, axis=\"x\")\nRHAND_Y_NAMES =  get_xy_names(RHAND_NAMES, axis=\"y\")\n\nLHAND_XY_NAMES = LHAND_X_NAMES + LHAND_Y_NAMES\nRHAND_XY_NAMES = RHAND_X_NAMES + RHAND_Y_NAMES\n\nLHAND_XY_IDXS = np.argwhere(np.isin(XY_POINT_LANDMARKS, LHAND_XY_NAMES)).squeeze()\nRHAND_XY_IDXS = np.argwhere(np.isin(XY_POINT_LANDMARKS, RHAND_XY_NAMES)).squeeze()","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:46.159878Z","iopub.execute_input":"2023-07-19T10:18:46.160266Z","iopub.status.idle":"2023-07-19T10:18:46.168872Z","shell.execute_reply.started":"2023-07-19T10:18:46.160233Z","shell.execute_reply":"2023-07-19T10:18:46.167769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Filter out too short frames","metadata":{}},{"cell_type":"code","source":"valid_df = train_df.copy()\nvalid_df[\"valid\"] = -1","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:50.830331Z","iopub.execute_input":"2023-07-19T10:18:50.830712Z","iopub.status.idle":"2023-07-19T10:18:50.849128Z","shell.execute_reply.started":"2023-07-19T10:18:50.830685Z","shell.execute_reply":"2023-07-19T10:18:50.847617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(valid_df['sequence_id'].unique()), len(train_df))","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:52.94796Z","iopub.execute_input":"2023-07-19T10:18:52.948309Z","iopub.status.idle":"2023-07-19T10:18:52.95839Z","shell.execute_reply.started":"2023-07-19T10:18:52.948281Z","shell.execute_reply":"2023-07-19T10:18:52.95738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, file_path in tqdm(enumerate(tqdm(valid_df['file_path'].unique()))):\n    df = pd.read_parquet(file_path)\n    for group, group_df in df.groupby('sequence_id'):\n        frames = group_df[XY_POINT_LANDMARKS].values\n        phrase_len = valid_df[valid_df.sequence_id==group][\"phrase_char_len\"]\n        \n        r_nonan = np.sum(np.sum(np.isnan(frames[:, RHAND_XY_IDXS]), axis = 1) == 0)\n        l_nonan = np.sum(np.sum(np.isnan(frames[:, LHAND_XY_IDXS]), axis = 1) == 0)\n        \n        no_nan = max(r_nonan, l_nonan)\n        if RATIO*len(phrase_len) <= no_nan:\n            valid_df.loc[valid_df[valid_df.sequence_id==group].index.values, \"valid\"] = 1\n        else:\n            valid_df.loc[valid_df[valid_df.sequence_id==group].index.values, \"valid\"] = 0","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:18:57.511052Z","iopub.execute_input":"2023-07-19T10:18:57.511606Z","iopub.status.idle":"2023-07-19T10:38:21.276272Z","shell.execute_reply.started":"2023-07-19T10:18:57.511546Z","shell.execute_reply":"2023-07-19T10:38:21.274017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert not (valid_df['valid']==-1).sum()\nassert len(np.unique(valid_df['valid'])) == 2\nprint(f\"#Total: {len(valid_df)}, #Too short frame: {(valid_df['valid']==0).sum()}, #Keep: {(valid_df['valid']==1).sum()}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:38:21.281531Z","iopub.execute_input":"2023-07-19T10:38:21.282154Z","iopub.status.idle":"2023-07-19T10:38:21.295409Z","shell.execute_reply.started":"2023-07-19T10:38:21.282116Z","shell.execute_reply":"2023-07-19T10:38:21.294641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_df_copy = valid_df.copy()\nvalid_df_copy.drop(valid_df_copy[valid_df_copy['valid'] == 0].index, inplace = True)\nvalid_df_copy.reset_index(drop=True, inplace=True)\nprint(len(valid_df_copy))","metadata":{"execution":{"iopub.status.busy":"2023-07-19T10:38:21.296585Z","iopub.execute_input":"2023-07-19T10:38:21.297509Z","iopub.status.idle":"2023-07-19T10:38:21.343118Z","shell.execute_reply.started":"2023-07-19T10:38:21.297481Z","shell.execute_reply":"2023-07-19T10:38:21.341353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Kfold split","metadata":{}},{"cell_type":"code","source":"class CFG:\n    seed = 42\n    n_splits = 10","metadata":{"execution":{"iopub.status.busy":"2023-07-17T11:16:37.438086Z","iopub.execute_input":"2023-07-17T11:16:37.438993Z","iopub.status.idle":"2023-07-17T11:16:37.445656Z","shell.execute_reply.started":"2023-07-17T11:16:37.438945Z","shell.execute_reply":"2023-07-17T11:16:37.444223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_df_copy[\"fold\"] = -1\n\nunique_phrases = valid_df_copy['phrase'].unique()\nnum_phrases = len(unique_phrases)\ninterval = num_phrases // CFG.n_splits + 1\n\nnp.random.seed(CFG.seed) \nnp.random.shuffle(unique_phrases)\nphrase_folds = []\nfor i in range(CFG.n_splits):\n    if i < CFG.n_splits - 1:\n        phrase_folds.append(unique_phrases[i*interval:(i+1)*interval])\n    else:\n        phrase_folds.append(unique_phrases[i*interval:])\n\nfor i, phrase_fold in enumerate(phrase_folds):\n    valid_df_copy.loc[valid_df_copy['phrase'].isin(phrase_fold), 'fold'] = i","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = 0\nfor i in range(CFG.n_splits):\n    count = len(valid_df_copy[valid_df_copy[\"fold\"] == i][\"phrase\"].unique())\n    print(count)\n    counts += count\nassert counts == len(valid_df_copy[\"phrase\"].unique())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_df_copy.to_csv(\"cleaned_train.csv\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save","metadata":{}},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=XY_POINT_LANDMARKS)","metadata":{"execution":{"iopub.status.busy":"2023-07-17T11:40:51.143933Z","iopub.execute_input":"2023-07-17T11:40:51.145021Z","iopub.status.idle":"2023-07-17T11:40:51.150943Z","shell.execute_reply.started":"2023-07-17T11:40:51.144969Z","shell.execute_reply":"2023-07-17T11:40:51.149884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(fold):\n    fold_df = valid_df_copy.loc[valid_df_copy.fold == fold]\n    tffile = f\"fold{fold}.tfrecord\"\n    print(f\"Saving {tffile}\")\n    with tf.io.TFRecordWriter(tffile) as file_writer:\n        for group, group_df in tqdm(fold_df.groupby('file_path')):\n            seqs = load_relevant_data_subset(group)\n            for index, row in group_df.iterrows():\n                frames = seqs[seqs.index == row[\"sequence_id\"]].to_numpy()\n                features = {XY_POINT_LANDMARKS[i]: tf.train.Feature(\n                    float_list=tf.train.FloatList(value=frames[:, i])) for i in range(len(XY_POINT_LANDMARKS))}\n                features[\"phrase\"] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[row[\"phrase\"].encode(\"utf-8\")]))\n                features[\"sequence_id\"] = tf.train.Feature(int64_list=tf.train.Int64List(value=[row[\"sequence_id\"]]))\n                record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n                file_writer.write(record_bytes)\n                del record_bytes","metadata":{"execution":{"iopub.status.busy":"2023-07-17T13:17:18.22389Z","iopub.execute_input":"2023-07-17T13:17:18.224439Z","iopub.status.idle":"2023-07-17T13:17:18.237115Z","shell.execute_reply.started":"2023-07-17T13:17:18.224385Z","shell.execute_reply":"2023-07-17T13:17:18.235297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = Parallel(n_jobs=cpu_count())(\n    delayed(process)(fold)\n    for fold in range(CFG.n_splits))","metadata":{"execution":{"iopub.status.busy":"2023-07-17T13:17:18.238899Z","iopub.execute_input":"2023-07-17T13:17:18.239383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}