{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Thanks for the public notebooks below:  \nhttps://www.kaggle.com/code/hoyso48/1st-place-solution-training  \nhttps://www.kaggle.com/code/irohith/aslfr-ctc-based-on-prev-comp-1st-place  \nhttps://www.kaggle.com/code/markwijkhuizen/aslfr-transformer-training-inference  \nThis is the 3rd place solution training code, you could refer the solution here:  \nhttps://www.kaggle.com/competitions/asl-fingerspelling/discussion/434393  ","metadata":{}},{"cell_type":"markdown","source":"# Install libs","metadata":{}},{"cell_type":"code","source":"!pip install -q icecream --no-index --find-links=file:///kaggle/input/icecream\n!pip install -q pymp-pypi --no-index --find-links=file:///kaggle/input/pymp-pypi/pymp-pypi-0.4.5/dist","metadata":{"execution":{"iopub.status.busy":"2023-08-26T04:35:57.874172Z","iopub.execute_input":"2023-08-26T04:35:57.874898Z","iopub.status.idle":"2023-08-26T04:36:18.993291Z","shell.execute_reply.started":"2023-08-26T04:35:57.874857Z","shell.execute_reply":"2023-08-26T04:36:18.992151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import libs","metadata":{}},{"cell_type":"code","source":"import sys, os\nimport numpy as np\nimport pandas as pd\nimport json\nimport re\nimport six\nfrom collections import Counter, OrderedDict, defaultdict\nfrom collections.abc import Iterable\nfrom multiprocessing import cpu_count\nfrom tqdm.notebook import tqdm\nfrom icecream import ic\nimport pymp\nimport tensorflow as tf\nic(tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-08-26T04:51:15.901847Z","iopub.execute_input":"2023-08-26T04:51:15.902353Z","iopub.status.idle":"2023-08-26T04:51:16.101783Z","shell.execute_reply.started":"2023-08-26T04:51:15.902284Z","shell.execute_reply":"2023-08-26T04:51:16.100987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Flags","metadata":{}},{"cell_type":"code","source":"class FLAGS(object):\n  # online==False means using n-fold split and train on fold 1,2, folds-1 while valid on fold 0\n  # online==True means using all train data but still will valid on fold 0\n  online = False  \n  folds = 4\n  fold_seed = 1229\n  root = '../input/asl-fingerspelling'\n  working = '/kaggle/working'\n  use_z = True  # use x,y,z if True\n  norm_frames = True # norm frames using x - mean / std\n  concat_frames = True # concat original and normalized frames\n  add_pos = True # add abs frame pos, like 1/1000., 2/1000.\n  sup_weight = 0.1 # for supplement dataset assigin weight 0.1\n\ndef load_json(filename):\n  with open(filename) as fh:\n    obj = json.load(fh)\n  return obj","metadata":{"execution":{"iopub.status.busy":"2023-08-26T04:36:28.333838Z","iopub.execute_input":"2023-08-26T04:36:28.33507Z","iopub.status.idle":"2023-08-26T04:36:28.341092Z","shell.execute_reply.started":"2023-08-26T04:36:28.335037Z","shell.execute_reply":"2023-08-26T04:36:28.340037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Common configs","metadata":{}},{"cell_type":"code","source":"LPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nLIP = [\n    61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n    291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n    95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\nic(len(LIP))\nLLIP = [84,181,91,146,61,185,40,39,37,87,178,88,95,78,191,80,81,82]\nRLIP = [314,405,321,375,291,409,270,269,267,317,402,318,324,308,415,310,311,312]\nMID_LIP = [i for i in LIP if i not in LLIP + RLIP]\nic(len(LLIP), len(RLIP), len(MID_LIP))\n\nNOSE=[\n    1,2,98,327\n]\nLNOSE = [98]\nRNOSE = [327]\nMID_NOSE = [i for i in NOSE if i not in LNOSE + RNOSE]\n\nLEYE = [\n    263, 249, 390, 373, 374, 380, 381, 382, 362,\n    466, 388, 387, 386, 385, 384, 398,\n]\nREYE = [\n    33, 7, 163, 144, 145, 153, 154, 155, 133,\n    246, 161, 160, 159, 158, 157, 173,\n]\n\nN_HAND_POINTS = 21\nN_POSE_POINTS = len(LPOSE)\nN_LIP_POINTS = len(LLIP)\nN_EYE_POINTS = len(LEYE)\nN_NOSE_POINTS = len(LNOSE)\nN_MID_POINTS = len(MID_LIP + MID_NOSE)\n\nSEL_COLS = []\nfor i in range(N_HAND_POINTS):\n  SEL_COLS.extend([f'x_left_hand_{i}', f'y_left_hand_{i}', f'z_left_hand_{i}'])\nfor i in range(N_HAND_POINTS):\n  SEL_COLS.extend([f'x_right_hand_{i}', f'y_right_hand_{i}', f'z_right_hand_{i}'])\nfor i in LPOSE:\n  SEL_COLS.extend([f'x_pose_{i}', f'y_pose_{i}', f'z_pose_{i}'])\nfor i in RPOSE:\n  SEL_COLS.extend([f'x_pose_{i}', f'y_pose_{i}', f'z_pose_{i}'])\nfor i in LLIP:\n  SEL_COLS.extend([f'x_face_{i}', f'y_face_{i}', f'z_face_{i}'])\nfor i in RLIP:\n  SEL_COLS.extend([f'x_face_{i}', f'y_face_{i}', f'z_face_{i}'])\n\nfor i in LEYE:\n  SEL_COLS.extend([f'x_face_{i}', f'y_face_{i}', f'z_face_{i}'])\nfor i in REYE:\n  SEL_COLS.extend([f'x_face_{i}', f'y_face_{i}', f'z_face_{i}'])\n  \nfor i in LNOSE:\n  SEL_COLS.extend([f'x_face_{i}', f'y_face_{i}', f'z_face_{i}'])\nfor i in RNOSE:\n  SEL_COLS.extend([f'x_face_{i}', f'y_face_{i}', f'z_face_{i}'])\n  \nfor i in MID_LIP:\n  SEL_COLS.extend([f'x_face_{i}', f'y_face_{i}', f'z_face_{i}'])\nfor i in MID_NOSE:\n  SEL_COLS.extend([f'x_face_{i}', f'y_face_{i}', f'z_face_{i}'])\n    \nN_COLS = len(SEL_COLS)\nic(N_COLS)\n    \nCHAR2IDX = load_json(f'../input/asl-fingerspelling/character_to_prediction_index.json')\nCHAR2IDX = {k: v + 1 for k, v in CHAR2IDX.items()}\nN_CHARS = len(CHAR2IDX)\nic(N_CHARS)\n\nPAD_IDX = 0\nSOS_IDX = PAD_IDX # Start Of Sentence\nEOS_IDX = N_CHARS + 1 # End Of Sentence\nic(PAD_IDX, SOS_IDX, EOS_IDX)\n\nPAD_TOKEN = '<PAD>'\nSOS_TOKEN = PAD_TOKEN\nEOS_TOKEN = '<EOS>'\n\nCHAR2IDX[PAD_TOKEN] = PAD_IDX\nCHAR2IDX[EOS_TOKEN] = EOS_IDX \n\nADDRESS_TOKEN = '<ADDRESS>'\nURL_TOKEN = '<URL>'\nPHONE_TOKEN = '<PHONE>'\nSUP_TOKEN = '<SUP>'\n\nVOCAB_SIZE = len(CHAR2IDX)\nIDX2CHAR = {v: k for k, v in CHAR2IDX.items()}\nic(VOCAB_SIZE)\nic(len(IDX2CHAR))\n\nSTATS = {}\nCLASSES = [\n  'address', \n  'url', \n  'phone', \n  'sup',\n  ]\nPHRASE_TYPES = dict(zip(CLASSES, range(len(CLASSES))))\nN_TYPES = len(CLASSES)\nMAX_PHRASE_LEN = 32\n\ndef get_vocab_size():\n  vocab_size = VOCAB_SIZE\n  return vocab_size\n\ndef get_n_cols(no_motion=False, use_z=None):\n  n_cols = N_COLS\n  if use_z is None:\n    use_z = FLAGS.use_z\n  \n  if FLAGS.concat_frames:\n    assert FLAGS.norm_frames\n    n_cols += N_COLS\n  \n  if not use_z:\n    n_cols = n_cols // 3 * 2\n    \n  if FLAGS.add_pos:\n    n_cols += 1\n  \n  return n_cols\n\ndef get_phrase_type(phrase):\n  # Phone Number\n  if re.match(r'^[\\d+-]+$', phrase):\n    return 'phone'\n  # url\n  elif any([substr in phrase for substr in ['www', '.', '/']\n           ]) and ' ' not in phrase:\n    return 'url'\n  # Address\n  else:\n    return 'address'","metadata":{"execution":{"iopub.status.busy":"2023-08-26T04:36:28.343418Z","iopub.execute_input":"2023-08-26T04:36:28.343707Z","iopub.status.idle":"2023-08-26T04:36:28.635112Z","shell.execute_reply.started":"2023-08-26T04:36:28.343683Z","shell.execute_reply":"2023-08-26T04:36:28.633957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess for tfrecords","metadata":{}},{"cell_type":"code","source":"def set_folds_(df, folds=5, group_key=None, stratify_key=None, seed=1024):\n  if stratify_key is None:\n    rng = np.random.default_rng(seed)\n    if group_key is not None:\n      group_values = df[group_key].unique()\n      ngroups = len(group_values)\n      x = np.arange(ngroups)\n      rng.shuffle(x)\n      xs = np.array_split(x, folds)\n      fold_values = np.asarray([0 for _ in range(ngroups)])\n      for fold, x in enumerate(xs):\n        fold_values[x] = fold\n      group2fold = dict(zip(group_values, fold_values))\n      df['fold'] = df[group_key].map(group2fold)\n    else:\n      fold_values = np.asarray([0 for _ in range(len(df))])\n      x = np.arange(len(df))\n      rng.shuffle(x)\n      xs = np.array_split(x, folds)\n      for fold, x in enumerate(xs):\n        fold_values[x] = fold\n      df['fold'] = fold_values\n  else:  \n    if group_key is None:\n      from sklearn.model_selection import StratifiedKFold\n      skf = StratifiedKFold(n_splits=folds, random_state=seed, shuffle=True)\n      folds = np.zeros(len(df), dtype=int)\n      splits = list(skf.split(df, df[stratify_key]))\n      for i, (_, val_idx) in enumerate(splits):\n        folds[val_idx] = i\n      df['fold'] = folds\n    else:\n      from sklearn.model_selection import StratifiedGroupKFold\n      skf = StratifiedGroupKFold(n_splits=folds, random_state=seed, shuffle=True)\n      folds = np.zeros(len(df), dtype=int)\n      splits = list(skf.split(df, df[stratify_key], df[group_key]))\n      for i, (_, val_idx) in enumerate(splits):\n        folds[val_idx] = i\n      df['fold'] = folds\n  return df\n\ndef init_folds_(train):\n  set_folds_(train, \n             FLAGS.folds,\n             group_key='participant_id', \n             stratify_key='phrase_type',\n             seed=FLAGS.fold_seed)","metadata":{"execution":{"iopub.status.busy":"2023-08-26T04:36:28.636269Z","iopub.execute_input":"2023-08-26T04:36:28.636531Z","iopub.status.idle":"2023-08-26T04:36:28.650041Z","shell.execute_reply.started":"2023-08-26T04:36:28.636507Z","shell.execute_reply":"2023-08-26T04:36:28.648939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_phrase_dup_(train):\n  counter = Counter()\n  for row in train.itertuples():\n    row = row._asdict()\n    phrase = row['phrase']\n    fold = row['fold']\n    counter[phrase] += 1\n    counter[f'{fold}^{phrase}'] += 1\n\n  l = []\n  for row in train.itertuples():\n    dup = 0\n    row = row._asdict()\n    phrase = row['phrase']\n    fold = row['fold']\n    if counter[f'{fold}^{phrase}'] < counter[phrase]:\n      dup = 1\n    l.append(dup)\n\n  train['phrase_dup'] = l\n  \ndef preprocess_parquet(file_path, save=False):\n  if save:\n    with open(f'{FLAGS.working}/inference_args.json', 'w') as f:\n      json.dump({ 'selected_columns': SEL_COLS }, f)\n  \n  df = pd.read_parquet(file_path, columns=SEL_COLS)\n  seq_ids = df.index.unique()\n  for seq_id in tqdm(seq_ids, total=len(seq_ids), desc='per_seq'):\n    frame = df[df.index == seq_id].values\n    assert frame.ndim == 2\n    assert frame.shape[-1] == N_COLS    \n    n_frame = frame.shape[0]\n    frame = list(frame.reshape(-1))\n    yield seq_id, frame, n_frame\n\ndef preprocss_(train):\n  train['phrase_len'] = train['phrase'].apply(len)\n  train['phrase_type'] = train['phrase'].apply(get_phrase_type)\n\n  # Get complete file path to file\n  def get_file_path(path):\n    return f'{FLAGS.root}/{path}'\n\n  train['file_path'] = train['path'].apply(get_file_path)\n\n  \ndef set_idx_(train):\n  idxes = [0] * FLAGS.folds\n  l = []\n  for row in train.itertuples():\n    l.append(idxes[row.fold])\n    idxes[row.fold] += 1\n  train['idx'] = l  \n\ndef init_dfs(obj='train'):\n  file_name = 'train' if obj == 'train' else 'supplemental_metadata'\n  train = pd.read_csv(f'{FLAGS.root}/{file_name}.csv')\n  preprocss_(train)\n  init_folds_(train)\n  check_phrase_dup_(train)\n  set_idx_(train)\n  return train","metadata":{"execution":{"iopub.status.busy":"2023-08-26T04:36:28.651639Z","iopub.execute_input":"2023-08-26T04:36:28.651906Z","iopub.status.idle":"2023-08-26T04:36:28.668031Z","shell.execute_reply.started":"2023-08-26T04:36:28.651882Z","shell.execute_reply":"2023-08-26T04:36:28.666907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gen tfrecords","metadata":{}},{"cell_type":"code","source":"def int_feature(value):\n  if not isinstance(value, (list, tuple)):\n    value = [value]\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n\n\ndef int64_feature(value):\n  if not isinstance(value, (list, tuple)):\n    value = [value]\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n\n\ndef bytes_feature(value):\n  if not isinstance(value, (list, tuple)):\n    value = [value]\n  if not six.PY2:\n    if isinstance(value[0], str):\n      value = [x.encode() for x in value]\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))\n\n\ndef float_feature(value):\n  if not isinstance(value, (list, tuple)):\n    value = [value]\n  return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef gen_feature(l, dtype=None):\n  if dtype is None:\n    if isinstance(l, (str, bytes)):\n      dtype = np.str_\n    elif isinstance(l, int):\n      dtype = np.int64\n    elif isinstance(l, float):\n      dtype = np.float32\n    else:\n      dtype = np.asarray(l).dtype\n\n  if isinstance(l, Iterable) and dtype != np.str_ and dtype != object:\n    l = list(l)\n\n  if dtype == object or dtype == np.str_:\n    try:\n      if l.startswith('(') and l.endswith(')') or l.startswith(\n          '[') and l.endswith(']'):\n        try:\n          l = l[1:-1].split(',')\n          l = [int(x.strip()) for x in l]\n          dtype = np.int64\n        except Exception:\n          pass\n    except Exception:\n      pass\n\n  if dtype == np.int64 or dtype == np.int32:\n    return int64_feature(l)\n  elif dtype == np.float32 or dtype == np.float64:\n    return float_feature(l)\n  elif dtype == object or dtype == np.str_ or dtype.str.startswith('<U'):\n    return bytes_feature(l)\n  else:\n    return bytes_feature(l)\n\ndef gen_features(feature, default_value=0):\n  feature_ = {}\n  for key in feature:\n    feature_[key] = feature[key]\n    if isinstance(feature[key], list or tuple) and not feature[key]:\n      feature_[key] = [default_value]\n  keys = list(feature_.keys())\n  for key in keys:\n    try:\n      feature_[key] = gen_feature(feature_[key])\n    except Exception as e:\n      del feature_[key]\n      # continue\n      print(e)\n      print('bad key', key)\n      exit(0)\n      # ic(e)\n      # raise (e)\n  return feature_\n\nclass TfrecordsWriter(object):\n  def __init__(self, filename, format='tfrec', buffer_size=None, \n               shuffle=False, seed=None, clear_first=False):\n    '''\n    buffer_size = None means write at once\n    = 0 means buffersize large engouh, only output at last \n    oterwise output when buffer full\n    '''\n    if seed:\n      self.rng = np.random.default_rng(seed)\n    self.count = 0\n    self.buffer_size = buffer_size\n    self.shuffle = shuffle\n    \n    fromat = filename.split('.')[-1]\n    assert filename.endswith('.' + format), f'file:{filename} format:{format}'\n    filename_ = filename[:-len(format)-1]\n    filename = filename_ + '.TMP'\n    dir_ = os.path.dirname(filename)\n    os.makedirs(dir_, exist_ok=True)\n\n    if clear_first:\n      command = f'rm -rf {dir_}/{filename_}.*.{format}'\n      ic(command)\n      os.system(command)\n    \n    self.writer = tf.io.TFRecordWriter(filename)\n    self.buffer = [] if self.buffer_size else None\n    self.sort_vals = []\n\n    self.filename = filename\n    self.format = format\n\n    self.closed = False\n\n  def __del__(self):\n    self.close()\n\n  def __enter__(self):\n    return self  \n\n  def __exit__(self, exc_type, exc_value, traceback):\n    self.close()\n\n  def close(self):\n    if not self.closed:\n      if self.buffer:\n        if self.shuffle:\n          self.rng.shuffle(self.buffer)\n        for example in self.buffer:\n          self.writer.write(example.SerializeToString())\n        self.buffer = []  \n        self.sort_vals = []\n\n      ifile = self.filename \n      if self.num_records:\n        ofile = ifile[:-len('.TMP')] + f'.{self.num_records}.{self.format}'\n        os.system(f'mv {ifile} {ofile}')\n      else:\n        print(f'removing {ifile}')\n        os.system(f'rm -rf {ifile}')\n      self.closed = True\n      self.count = 0\n    \n  def finalize(self):\n    self.close()\n    \n  def write(self, feature, sort_val=None):\n    self.write_feature(feature, sort_val)\n\n  def write_feature(self, feature, sort_key=None):\n    fe = gen_features(feature)\n    example = tf.train.Example(features=tf.train.Features(feature=fe))\n    if sort_key is None:\n      self.write_example(example)\n    else:\n      self.write_example(example, feature[sort_key])\n\n  def write_example(self, example, sort_val=None):\n    self.count += 1\n    if self.buffer is not None:\n      self.buffer.append(example)\n      if sort_val is not None:\n        self.sort_vals.append(sort_val)\n      if len(self.buffer) >= self.buffer_size and self.buffer_size != 0:\n        if self.sort_vals:\n          assert self.buffer_size == 0, 'sort all values require buffer_size==0'\n          yx = zip(self.sort_vals, self.buffer)\n          yx.sort()\n          self.buffer = [x for y, x in yx]\n        elif self.shuffle: # if sort_vals not do shuffle anymore\n          self.rng.shuffle(self.buffer)\n        for example in self.buffer:\n          self.writer.write(example.SerializeToString())\n        self.buffer = []\n    else:\n      self.writer.write(example.SerializeToString())\n\n  def size(self):\n    return self.count\n\n  @property\n  def num_records(self):\n    return self.count","metadata":{"execution":{"iopub.status.busy":"2023-08-26T04:36:28.669522Z","iopub.execute_input":"2023-08-26T04:36:28.669822Z","iopub.status.idle":"2023-08-26T04:36:28.699802Z","shell.execute_reply.started":"2023-08-26T04:36:28.669797Z","shell.execute_reply":"2023-08-26T04:36:28.698573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = {}\nfile_paths = []\nrecords_dir = None\n\ndef pad(l, max_len, pad_idx=0):\n  if len(l) >= max_len:\n    return l[:max_len]\n  else:\n    l = l + [pad_idx] * (max_len - len(l))\n    return l\n\ndef gen_record(index, obj):\n  file_path = file_paths[index]\n  start_idx = index * FLAGS.folds\n  ofiles = [f'{records_dir}/{start_idx + idx}.tfrec' for idx in range(FLAGS.folds)]\n  writers = [TfrecordsWriter(ofile, buffer_size=1000, shuffle=True, seed=1024) for ofile in ofiles]\n  for sequence_id, frame, n_frame in preprocess_parquet(file_path, save=(index==0)):\n    row = train[sequence_id]\n    fe = {}\n    for key in ['sequence_id', 'file_id', 'participant_id', 'phrase', 'fold',\n                'phrase_len', 'phrase_type', 'phrase_dup', 'idx']:\n      fe[key] = row[key]\n    fe['frames'] = frame\n    fe['n_frames'] = n_frame\n    fe['frame_mean'] = np.nan_to_num(np.array(frame)).mean()\n    fe['n_frames_per_char'] = n_frame / row['phrase_len']\n    \n    phrase = [CHAR2IDX[c] for c in row['phrase']]\n    \n    # ignore 0 for pad, so need -1\n    fe['first_char'] = phrase[0] - 1\n    fe['last_char'] = phrase[-1] - 1\n    phrase.append(EOS_IDX)\n    phrase = pad(phrase, MAX_PHRASE_LEN, PAD_IDX)\n    fe['phrase'] = row['phrase']\n    fe['phrase_'] = phrase\n    fe['phrase_type_'] = PHRASE_TYPES[row['phrase_type']]\n    fe['weight'] = 1.0 if obj == 'train' else FLAGS.sup_weight\n    \n    cls_label = [0] * N_CHARS\n    for c in row['phrase']:\n      cls_label[CHAR2IDX[c] - 1] = 1\n    fe['cls_label'] = cls_label\n    \n    writers[row['fold']].write(fe)\n    \n  for writer in writers:\n    writer.close()\n    \ndef gen_records(obj, out_dir):\n  global records_dir\n  df = init_dfs(obj=obj)\n  records_dir = f'{FLAGS.working}/tfrecords/{out_dir}'\n  ic(records_dir)\n  os.system(f'mkdir -p {records_dir}')\n  for row in tqdm(df.itertuples(), total=len(df), desc='train'):\n    row = row._asdict()\n    train[row['sequence_id']] = row\n\n  file_paths.extend(df.file_path.unique())\n  num_records = len(file_paths)\n  ic(num_records)\n  num_workers = cpu_count()\n  inputs_list = np.array_split(list(range(num_records)), num_workers)\n  with pymp.Parallel(num_workers) as p:\n    for i in p.range(num_workers):\n      for x in tqdm(inputs_list[i], desc='gen_records'):\n        gen_record(x, obj)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-26T04:51:16.103702Z","iopub.execute_input":"2023-08-26T04:51:16.103964Z","iopub.status.idle":"2023-08-26T04:51:16.219298Z","shell.execute_reply.started":"2023-08-26T04:51:16.103941Z","shell.execute_reply":"2023-08-26T04:51:16.218112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen_records(obj='train', out_dir='train')","metadata":{"execution":{"iopub.status.busy":"2023-08-26T04:36:28.725097Z","iopub.execute_input":"2023-08-26T04:36:28.725545Z","iopub.status.idle":"2023-08-26T04:51:15.895238Z","shell.execute_reply.started":"2023-08-26T04:36:28.725518Z","shell.execute_reply":"2023-08-26T04:51:15.892853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## this is for generating suplement dataset tfrecords which is used for training also, here commented out for output size over limit\n#gen_records(obj='sup', out_dir='sup')","metadata":{"execution":{"iopub.status.busy":"2023-08-26T04:51:27.656689Z","iopub.status.idle":"2023-08-26T04:51:27.657127Z","shell.execute_reply.started":"2023-08-26T04:51:27.656932Z","shell.execute_reply":"2023-08-26T04:51:27.656949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}