{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook prepares the tfrecords for my simple transformer notebook and is almost an identical copy of Rohith Ingilela's [notebook](https://www.kaggle.com/code/irohith/aslfr-preprocess-dataset), with two small changes according to his latest update and remark:\n1. Only hands landmarks + ten pose landmarks.\n2. Only movies with a number of frames > twice the length of the phrase.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom skimage.transform import resize\nimport json\nfrom tqdm import tqdm\nimport os\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-06-15T19:31:16.73801Z","iopub.execute_input":"2023-06-15T19:31:16.738704Z","iopub.status.idle":"2023-06-15T19:31:25.917954Z","shell.execute_reply.started":"2023-06-15T19:31:16.738659Z","shell.execute_reply":"2023-06-15T19:31:25.916524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inpdir = \"/kaggle/input/asl-fingerspelling\"\ndf = pd.read_csv(f'{inpdir}/train.csv')\ndf[\"phrase_bytes\"] = df[\"phrase\"].map(lambda x: x.encode(\"utf-8\"))\ndisplay(df.head())","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:41:25.289121Z","iopub.execute_input":"2023-06-15T09:41:25.289761Z","iopub.status.idle":"2023-06-15T09:41:25.498159Z","shell.execute_reply.started":"2023-06-15T09:41:25.289726Z","shell.execute_reply":"2023-06-15T09:41:25.497293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nRHAND_LBLS = [f'x_right_hand_{i}' for i in range(21)] + [f'y_right_hand_{i}' for i in range(21)] + [f'z_right_hand_{i}' for i in range(21)]\nLHAND_LBLS = [ f'x_left_hand_{i}' for i in range(21)] + [ f'y_left_hand_{i}' for i in range(21)] + [ f'z_left_hand_{i}' for i in range(21)]\nPOSE_LBLS = [f'x_pose_{i}' for i in POSE] + [f'y_pose_{i}' for i in POSE] + [f'z_pose_{i}' for i in POSE]\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE]\n\nSEL_COLS = X + Y + Z\nFRAME_LEN = 128\n\nX_IDX = [i for i, col in enumerate(SEL_COLS) if \"x_\" in col]\nY_IDX = [i for i, col in enumerate(SEL_COLS) if \"y_\" in col]\nZ_IDX = [i for i, col in enumerate(SEL_COLS) if \"z_\" in col]\n\nRHAND_IDX = [i for i, col in enumerate(SEL_COLS) if \"right\" in col]\nLHAND_IDX = [i for i, col in enumerate(SEL_COLS) if \"left\" in col]\nRPOSE_IDX = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in RPOSE]\nLPOSE_IDX = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in LPOSE]\n\nprint('SEL_COLS size:' + str(len(SEL_COLS)))","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:41:25.499223Z","iopub.execute_input":"2023-06-15T09:41:25.500302Z","iopub.status.idle":"2023-06-15T09:41:25.51535Z","shell.execute_reply.started":"2023-06-15T09:41:25.500258Z","shell.execute_reply":"2023-06-15T09:41:25.514065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_landmarks = pd.read_parquet('/kaggle/input/asl-fingerspelling/train_landmarks/1019715464.parquet')\nkeys = train_landmarks.keys()[1:]\ntrain_landmarks.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:41:25.517724Z","iopub.execute_input":"2023-06-15T09:41:25.518152Z","iopub.status.idle":"2023-06-15T09:41:40.327515Z","shell.execute_reply.started":"2023-06-15T09:41:25.518117Z","shell.execute_reply":"2023-06-15T09:41:40.326523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nnp.unique([keys[i].split('_')[1] for i in range(len(keys))])\nlen(keys)/3\n[keys[i].split('_') for i in range(len(keys)) if 'face' in keys[i].split('_')]\n'''","metadata":{"execution":{"iopub.status.busy":"2023-06-15T09:41:40.329951Z","iopub.execute_input":"2023-06-15T09:41:40.330927Z","iopub.status.idle":"2023-06-15T09:41:40.337332Z","shell.execute_reply.started":"2023-06-15T09:41:40.330874Z","shell.execute_reply":"2023-06-15T09:41:40.336484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n return pd.read_parquet(pq_path, columns=SEL_COLS)\n\ncounter = 0\nfor file_id in tqdm(df.file_id.unique()):\n \n print(counter)\n counter+=1\n \n pqfile = f\"{inpdir}/train_landmarks/{file_id}.parquet\"\n if not os.path.isdir(\"tfds\"): os.mkdir(\"tfds\")\n tffile = f\"tfds/{file_id}.tfrecord\"\n seq_refs = df.loc[df.file_id == file_id]\n seqs = load_relevant_data_subset(pqfile)\n seqs_numpy = seqs.to_numpy()\n with tf.io.TFRecordWriter(tffile) as file_writer:\n for seq_id, phrase in zip(seq_refs.sequence_id, seq_refs.phrase_bytes):\n frames = seqs_numpy[seqs.index == seq_id]\n \n r_nonan = np.sum(np.sum(np.isnan(frames[:, RHAND_IDX]), axis = 1) == 0)\n l_nonan = np.sum(np.sum(np.isnan(frames[:, LHAND_IDX]), axis = 1) == 0)\n no_nan = max(r_nonan, l_nonan)\n \n if 2*len(phrase)