{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport json\nimport math\nimport pickle\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow_addons as tfa","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:29.615724Z","iopub.execute_input":"2023-08-23T15:11:29.616627Z","iopub.status.idle":"2023-08-23T15:11:42.974021Z","shell.execute_reply.started":"2023-08-23T15:11:29.616561Z","shell.execute_reply":"2023-08-23T15:11:42.972815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_strategy(device='TPU-VM'):\n if \"TPU\" in device:\n try:\n tpu = 'local' if device == 'TPU-VM' else None\n print(\"Connecting to TPU...\")\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=tpu)\n strategy = tf.distribute.TPUStrategy(tpu)\n IS_TPU = True\n print(\"Connected to TPU.\")\n except:\n print(\"TPU not available. Falling back to GPU...\")\n device = \"GPU\"\n\n if device == \"GPU\":\n IS_TPU = False\n ngpu = len(tf.config.experimental.list_physical_devices('GPU'))\n if ngpu > 1:\n print(\"Using multiple GPUs.\")\n strategy = tf.distribute.MirroredStrategy()\n elif ngpu == 1:\n print(\"Using single GPU.\")\n strategy = tf.distribute.OneDeviceStrategy(\"GPU:0\")\n else:\n print(\"No GPU available. Falling back to CPU...\")\n device = \"CPU\"\n\n if device == \"CPU\":\n print(\"Using CPU.\")\n strategy = tf.distribute.OneDeviceStrategy(\"CPU\")\n IS_TPU = False\n\n AUTO = tf.data.experimental.AUTOTUNE\n REPLICAS = strategy.num_replicas_in_sync\n print(f\"REPLICAS: {REPLICAS}\")\n\n return strategy, REPLICAS, IS_TPU\n\nSTRATEGY, N_REPLICAS, IS_TPU = get_strategy()\n\nLOAD_WEIGHTS = False","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:42.975928Z","iopub.execute_input":"2023-08-23T15:11:42.976679Z","iopub.status.idle":"2023-08-23T15:11:43.389769Z","shell.execute_reply.started":"2023-08-23T15:11:42.976643Z","shell.execute_reply":"2023-08-23T15:11:43.388513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n char_to_num = json.load(f)\n\npad_token = '^'\npad_token_idx = 59\n\nchar_to_num[pad_token] = pad_token_idx\n\nnum_to_char = {j:i for i,j in char_to_num.items()}\ndf = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n\nLIP = [\n 61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n 291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n 78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n 95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\nLPOSE = [13, 15, 17, 19, 21]\nRPOSE = [14, 16, 18, 20, 22]\nPOSE = LPOSE + RPOSE\n\nX = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE] + [f'x_face_{i}' for i in LIP]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE] + [f'y_face_{i}' for i in LIP]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE] + [f'z_face_{i}' for i in LIP]\n\nSEL_COLS = X + Y + Z\nFRAME_LEN = 128\nMAX_PHRASE_LENGTH = 64\n\nLIP_IDX_X = [i for i, col in enumerate(SEL_COLS) if \"face\" in col and \"x\" in col]\nRHAND_IDX_X = [i for i, col in enumerate(SEL_COLS) if \"right\" in col and \"x\" in col]\nLHAND_IDX_X = [i for i, col in enumerate(SEL_COLS) if \"left\" in col and \"x\" in col]\nRPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in RPOSE and \"x\" in col]\nLPOSE_IDX_X = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in LPOSE and \"x\" in col]\n\nLIP_IDX_Y = [i for i, col in enumerate(SEL_COLS) if \"face\" in col and \"y\" in col]\nRHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS) if \"right\" in col and \"y\" in col]\nLHAND_IDX_Y = [i for i, col in enumerate(SEL_COLS) if \"left\" in col and \"y\" in col]\nRPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in RPOSE and \"y\" in col]\nLPOSE_IDX_Y = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in LPOSE and \"y\" in col]\n\nLIP_IDX_Z = [i for i, col in enumerate(SEL_COLS) if \"face\" in col and \"z\" in col]\nRHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS) if \"right\" in col and \"z\" in col]\nLHAND_IDX_Z = [i for i, col in enumerate(SEL_COLS) if \"left\" in col and \"z\" in col]\nRPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in RPOSE and \"z\" in col]\nLPOSE_IDX_Z = [i for i, col in enumerate(SEL_COLS) if \"pose\" in col and int(col[-2:]) in LPOSE and \"z\" in col]","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:43.392082Z","iopub.execute_input":"2023-08-23T15:11:43.393519Z","iopub.status.idle":"2023-08-23T15:11:43.649504Z","shell.execute_reply.started":"2023-08-23T15:11:43.393459Z","shell.execute_reply":"2023-08-23T15:11:43.64821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUPPLEMENTAL_RHM = np.load(\"/kaggle/input/supplemently-data-set/mean_std/rh_mean.npy\")\nSUPPLEMENTAL_LHM = np.load(\"/kaggle/input/supplemently-data-set/mean_std/lh_mean.npy\")\nSUPPLEMENTAL_RPM = np.load(\"/kaggle/input/supplemently-data-set/mean_std/rp_mean.npy\")\nSUPPLEMENTAL_LPM = np.load(\"/kaggle/input/supplemently-data-set/mean_std/lp_mean.npy\")\nSUPPLEMENTAL_LIPM = np.load(\"/kaggle/input/supplemently-data-set/mean_std/lip_mean.npy\")\n\nSUPPLEMENTAL_RHS = np.load(\"/kaggle/input/supplemently-data-set/mean_std/rh_std.npy\")\nSUPPLEMENTAL_LHS = np.load(\"/kaggle/input/supplemently-data-set/mean_std/lh_std.npy\")\nSUPPLEMENTAL_RPS = np.load(\"/kaggle/input/supplemently-data-set/mean_std/rp_std.npy\")\nSUPPLEMENTAL_LPS = np.load(\"/kaggle/input/supplemently-data-set/mean_std/lp_std.npy\")\nSUPPLEMENTAL_LIPS = np.load(\"/kaggle/input/supplemently-data-set/mean_std/lip_std.npy\")\n\nTRAIN_RHM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rh_mean.npy\")\nTRAIN_LHM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lh_mean.npy\")\nTRAIN_RPM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rp_mean.npy\")\nTRAIN_LPM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lp_mean.npy\")\nTRAIN_LIPM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lip_mean.npy\")\n\nTRAIN_RHS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rh_std.npy\")\nTRAIN_LHS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lh_std.npy\")\nTRAIN_RPS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rp_std.npy\")\nTRAIN_LPS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lp_std.npy\")\nTRAIN_LIPS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lip_std.npy\")\n\n\nRHM = (TRAIN_RHM + SUPPLEMENTAL_RHM) / 2\nLHM = (TRAIN_LHM + SUPPLEMENTAL_LHM) / 2\nRPM = (TRAIN_RPM + SUPPLEMENTAL_RPM) / 2\nLPM = (TRAIN_LPM + SUPPLEMENTAL_LPM) / 2\nLIPM = (TRAIN_LIPM + SUPPLEMENTAL_LIPM) / 2\n\nRHS = (TRAIN_RHS + SUPPLEMENTAL_RHS) / 2\nLHS = (TRAIN_LHS + SUPPLEMENTAL_LHS) / 2\nRPS = (TRAIN_RPS + SUPPLEMENTAL_RPS) / 2\nLPS = (TRAIN_LPS + SUPPLEMENTAL_LPS) / 2\nLIPS = (TRAIN_LIPS + SUPPLEMENTAL_LIPS) / 2\n\n\n\n\n\n\"\"\"RHM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rh_mean.npy\")\nLHM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lh_mean.npy\")\nRPM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rp_mean.npy\")\nLPM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lp_mean.npy\")\nLIPM = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lip_mean.npy\")\n\nRHS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rh_std.npy\")\nLHS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lh_std.npy\")\nRPS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/rp_std.npy\")\nLPS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lp_std.npy\")\nLIPS = np.load(\"/kaggle/input/aslfr-dataset-tfrecords/mean_std/lip_std.npy\")\"\"\"\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:43.654035Z","iopub.execute_input":"2023-08-23T15:11:43.654901Z","iopub.status.idle":"2023-08-23T15:11:43.875342Z","shell.execute_reply.started":"2023-08-23T15:11:43.654848Z","shell.execute_reply":"2023-08-23T15:11:43.873746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n return pd.read_parquet(pq_path, columns=SEL_COLS)\n\nfile_id = df.file_id.iloc[0]\ninpdir = \"/kaggle/input/asl-fingerspelling/train_landmarks\"\npqfile = f\"{inpdir}/{file_id}.parquet\"\nseq_refs = df.loc[df.file_id == file_id]\nseqs = load_relevant_data_subset(pqfile)\n\nseq_id = seq_refs.sequence_id.iloc[0]\nframes = seqs.iloc[seqs.index == seq_id]\nphrase = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:43.877286Z","iopub.execute_input":"2023-08-23T15:11:43.878353Z","iopub.status.idle":"2023-08-23T15:11:47.328996Z","shell.execute_reply.started":"2023-08-23T15:11:43.878292Z","shell.execute_reply":"2023-08-23T15:11:47.327887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n@tf.function()\ndef resize_pad(x):\n if tf.shape(x)[0] < FRAME_LEN:\n x = tf.pad(x, ([[0, FRAME_LEN-tf.shape(x)[0]], [0, 0], [0, 0]]), constant_values=float(\"NaN\"))\n else:\n x = tf.image.resize(x, (FRAME_LEN, tf.shape(x)[1]))\n return x\n\n@tf.function(jit_compile=True)\ndef pre_process0(x):\n lip_x = tf.gather(x, LIP_IDX_X, axis=1)\n lip_y = tf.gather(x, LIP_IDX_Y, axis=1)\n lip_z = tf.gather(x, LIP_IDX_Z, axis=1)\n\n rhand_x = tf.gather(x, RHAND_IDX_X, axis=1)\n rhand_y = tf.gather(x, RHAND_IDX_Y, axis=1)\n rhand_z = tf.gather(x, RHAND_IDX_Z, axis=1)\n \n lhand_x = tf.gather(x, LHAND_IDX_X, axis=1)\n lhand_y = tf.gather(x, LHAND_IDX_Y, axis=1)\n lhand_z = tf.gather(x, LHAND_IDX_Z, axis=1)\n\n rpose_x = tf.gather(x, RPOSE_IDX_X, axis=1)\n rpose_y = tf.gather(x, RPOSE_IDX_Y, axis=1)\n rpose_z = tf.gather(x, RPOSE_IDX_Z, axis=1)\n \n lpose_x = tf.gather(x, LPOSE_IDX_X, axis=1)\n lpose_y = tf.gather(x, LPOSE_IDX_Y, axis=1)\n lpose_z = tf.gather(x, LPOSE_IDX_Z, axis=1)\n \n lip = tf.concat([lip_x[..., tf.newaxis], lip_y[..., tf.newaxis], lip_z[..., tf.newaxis]], axis=-1)\n rhand = tf.concat([rhand_x[..., tf.newaxis], rhand_y[..., tf.newaxis], rhand_z[..., tf.newaxis]], axis=-1)\n lhand = tf.concat([lhand_x[..., tf.newaxis], lhand_y[..., tf.newaxis], lhand_z[..., tf.newaxis]], axis=-1)\n rpose = tf.concat([rpose_x[..., tf.newaxis], rpose_y[..., tf.newaxis], rpose_z[..., tf.newaxis]], axis=-1)\n lpose = tf.concat([lpose_x[..., tf.newaxis], lpose_y[..., tf.newaxis], lpose_z[..., tf.newaxis]], axis=-1)\n \n hand = tf.concat([rhand, lhand], axis=1)\n hand = tf.where(tf.math.is_nan(hand), 0.0, hand)\n mask = tf.math.not_equal(tf.reduce_sum(hand, axis=[1, 2]), 0.0)\n\n lip = lip[mask]\n rhand = rhand[mask]\n lhand = lhand[mask]\n rpose = rpose[mask]\n lpose = lpose[mask]\n\n return rhand, lhand, lpose, rpose, lip\n\ntf.function()\ndef pre_process1(rhand, lhand, lpose, rpose, lip):\n lip = (resize_pad(lip) - LIPM) / LIPS\n rhand = (resize_pad(rhand) - RHM) / RHS\n lhand = (resize_pad(lhand) - LHM) / LHS\n rpose = (resize_pad(rpose) - RPM) / RPS\n lpose = (resize_pad(lpose) - LPM) / LPS\n\n\n #if augment:\n #print('augment')\n #rhand, lhand, lpose, rpose, lip = augment_fn((rhand, lhand, lpose, rpose, lip), always=always, max_len=max_len)\n \n x = tf.concat([rhand, lhand, lpose, rpose, lip], axis=1)\n s = tf.shape(x)\n #x = tf.reshape(x, (s[0], s[1]*s[2]))\n x = tf.concat([x[..., 0], x[..., 1], x[..., 2]], axis=-1)\n x = tf.where(tf.math.is_nan(x), 0.0, x)\n return x\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:47.332537Z","iopub.execute_input":"2023-08-23T15:11:47.333126Z","iopub.status.idle":"2023-08-23T15:11:47.359109Z","shell.execute_reply.started":"2023-08-23T15:11:47.333084Z","shell.execute_reply":"2023-08-23T15:11:47.357735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n\"\"\"def compute_velocity(x):\n # Calculate velocity: difference between consecutive frames\n dx = x[:, 1:] - x[:, :-1]\n # Pad zeros to maintain original shape\n dx_padded = tf.pad(dx, [[0, 0], [0, 1], [0, 0]])\n\n return dx_padded\n\ndef compute_acceleration(x):\n # Calculate acceleration: difference between consecutive velocities\n dv = x[:, 1:] - x[:, :-1]\n # Pad zeros to maintain original shape\n dv_padded = tf.pad(dv, [[0, 0], [0, 1], [0, 0]])\n\n return dv_padded\n\ndef get_vel_acc(landmarks):\n velocity = compute_velocity(landmarks)\n acceleration = compute_acceleration(velocity)\n\n # Concatenate original landmarks, velocity, and acceleration\n processed_landmarks = tf.concat([landmarks, velocity, acceleration], axis=1)\n\n return processed_landmarks\n\n\n\n\ndef tf_nan_mean(x, axis=0, keepdims=False):\n return tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis, keepdims=keepdims) / tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis, keepdims=keepdims)\n\ndef tf_nan_std(x, center=None, axis=0, keepdims=False):\n if center is None:\n center = tf_nan_mean(x, axis=axis, keepdims=True)\n d = x - center\n return tf.math.sqrt(tf_nan_mean(d * d, axis=axis, keepdims=keepdims))\n\n\n\n@tf.function()\ndef pre_process1(rhand, lhand, lpose, rpose, lip):\n lip = (resize_pad(lip) - tf_nan_mean(lip,axis=[0,1], keepdims=True)) / tf_nan_std(lip,axis=[0,1], keepdims=True)\n rhand = (resize_pad(rhand) - tf_nan_mean(rhand,axis=[0,1], keepdims=True)) / tf_nan_std(rhand,axis=[0,1], keepdims=True)\n lhand = (resize_pad(lhand) - tf_nan_mean(lhand,axis=[0,1], keepdims=True)) / tf_nan_std(lhand,axis=[0,1], keepdims=True)\n rpose = (resize_pad(rpose) - tf_nan_mean(rpose,axis=[0,1], keepdims=True)) / tf_nan_std(rpose,axis=[0,1], keepdims=True)\n lpose = (resize_pad(lpose) - tf_nan_mean(lpose,axis=[0,1], keepdims=True)) / tf_nan_std(lpose,axis=[0,1], keepdims=True)\n \n #if augment:\n #print('augment')\n #rhand, lhand, lpose, rpose, lip = augment_fn((rhand, lhand, lpose, rpose, lip), always=always, max_len=max_len)\n \n x = tf.concat([rhand, lhand, lpose, rpose, lip], axis=1)\n #x = get_vel_acc(x)\n s = tf.shape(x)\n #x = tf.reshape(x, (s[0], s[1]*s[2]))\n x = tf.concat([x[..., 0], x[..., 1], x[..., 2]], axis=-1)\n x = tf.where(tf.math.is_nan(x), 0.0, x)\n return x\"\"\"\n\ninput_shpae=[12,3]","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:47.360927Z","iopub.execute_input":"2023-08-23T15:11:47.361428Z","iopub.status.idle":"2023-08-23T15:11:47.384317Z","shell.execute_reply.started":"2023-08-23T15:11:47.361394Z","shell.execute_reply":"2023-08-23T15:11:47.383329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_fn(record_bytes):\n schema = {\n \"lip\": tf.io.VarLenFeature(tf.float32),\n \"rhand\": tf.io.VarLenFeature(tf.float32),\n \"lhand\": tf.io.VarLenFeature(tf.float32),\n \"rpose\": tf.io.VarLenFeature(tf.float32),\n \"lpose\": tf.io.VarLenFeature(tf.float32),\n \"phrase\": tf.io.VarLenFeature(tf.int64)\n }\n x = tf.io.parse_single_example(record_bytes, schema)\n\n lip = tf.reshape(tf.sparse.to_dense(x[\"lip\"]), (-1, 40, 3))\n rhand = tf.reshape(tf.sparse.to_dense(x[\"rhand\"]), (-1, 21, 3))\n lhand = tf.reshape(tf.sparse.to_dense(x[\"lhand\"]), (-1, 21, 3))\n rpose = tf.reshape(tf.sparse.to_dense(x[\"rpose\"]), (-1, 5, 3))\n lpose = tf.reshape(tf.sparse.to_dense(x[\"lpose\"]), (-1, 5, 3))\n phrase = tf.sparse.to_dense(x[\"phrase\"])\n\n return lip, rhand, lhand, rpose, lpose, phrase\n\ndef pre_process_fn(lip, rhand, lhand, rpose, lpose, phrase):\n phrase = tf.pad(phrase, [[0, MAX_PHRASE_LENGTH-tf.shape(phrase)[0]]], constant_values=pad_token_idx)\n return pre_process1(rhand, lhand, lpose, rpose, lip), phrase\n \ntffiles = [f\"/kaggle/input/aslfr-dataset-tfrecords/tfds/{file_id}.tfrecord\" for file_id in df.file_id.unique()]\nval_len = 1#int(0.05 * len(pqfiles))\ntrain_batch_size = 32\nval_batch_size = 32\n\ntrain_dataset = tf.data.TFRecordDataset(tffiles[val_len:]).prefetch(tf.data.AUTOTUNE).shuffle(5000).map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE).map(pre_process_fn, num_parallel_calls=tf.data.AUTOTUNE).batch(train_batch_size).prefetch(tf.data.AUTOTUNE)\nval_dataset = tf.data.TFRecordDataset(tffiles[:val_len]).prefetch(tf.data.AUTOTUNE).map(decode_fn, num_parallel_calls=tf.data.AUTOTUNE).map(pre_process_fn, num_parallel_calls=tf.data.AUTOTUNE).batch(val_batch_size).prefetch(tf.data.AUTOTUNE)\n\nbatch = next(iter(val_dataset))\nbatch[0].shape, batch[1].shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:47.385958Z","iopub.execute_input":"2023-08-23T15:11:47.387259Z","iopub.status.idle":"2023-08-23T15:11:48.93651Z","shell.execute_reply.started":"2023-08-23T15:11:47.387215Z","shell.execute_reply":"2023-08-23T15:11:48.935315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"foo=next(iter(train_dataset))","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:48.938144Z","iopub.execute_input":"2023-08-23T15:11:48.939222Z","iopub.status.idle":"2023-08-23T15:11:56.109222Z","shell.execute_reply.started":"2023-08-23T15:11:48.939176Z","shell.execute_reply":"2023-08-23T15:11:56.107201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LAYER_NORM_EPS = 1e-6\n\n# Dropout\nEMBEDDING_DROPOUT = 0.15\n\nMLP_RATIO = 2\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:56.116021Z","iopub.execute_input":"2023-08-23T15:11:56.117069Z","iopub.status.idle":"2023-08-23T15:11:56.124483Z","shell.execute_reply.started":"2023-08-23T15:11:56.117022Z","shell.execute_reply":"2023-08-23T15:11:56.123194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scaled_dot_product(q,k,v, softmax, attention_mask):\n #calculates Q . K(transpose)\n qkt = tf.matmul(q,k,transpose_b=True)\n #caculates scaling factor\n dk = tf.math.sqrt(tf.cast(q.shape[-1],dtype=tf.float32))\n scaled_qkt = qkt/dk\n softmax = softmax(scaled_qkt, mask=attention_mask)\n z = tf.matmul(softmax,v)\n #shape: (m,Tx,depth), same shape as q,k,v\n return z\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n def __init__(self, d_model, num_of_heads,dropout):\n super(MultiHeadAttention, self).__init__()\n self.d_model = d_model\n self.num_of_heads = num_of_heads\n self.depth = d_model // num_of_heads\n self.wq = tf.keras.layers.Dense(d_model)\n self.wk = tf.keras.layers.Dense(d_model)\n self.wv = tf.keras.layers.Dense(d_model)\n self.wo = tf.keras.layers.Dense(d_model)\n self.softmax = tf.keras.layers.Softmax()\n self.dropout = tf.keras.layers.Dropout(dropout)\n\n def call(self, q, k, v, attention_mask=None):\n multi_attn = []\n for i in range(self.num_of_heads):\n Q = self.wq(q)\n K = self.wk(k)\n V = self.wv(v)\n multi_attn.append(scaled_dot_product(Q, K, V, self.softmax, attention_mask))\n\n multi_head = tf.concat(multi_attn, axis=-1)\n multi_head_attention = self.wo(multi_head)\n multi_head_attention = self.dropout(multi_head_attention)\n return multi_head_attention","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:11:56.127822Z","iopub.execute_input":"2023-08-23T15:11:56.128768Z","iopub.status.idle":"2023-08-23T15:11:56.153096Z","shell.execute_reply.started":"2023-08-23T15:11:56.128722Z","shell.execute_reply":"2023-08-23T15:11:56.151696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ECA(tf.keras.layers.Layer):\n def __init__(self, kernel_size=5, **kwargs):\n super().__init__(**kwargs)\n self.supports_masking = True\n self.kernel_size = kernel_size\n self.conv = tf.keras.layers.Conv1D(1, kernel_size=kernel_size, strides=1, padding=\"same\", use_bias=False)\n\n def call(self, inputs, mask=None):\n nn = tf.keras.layers.GlobalAveragePooling1D()(inputs, mask=mask)\n nn = tf.expand_dims(nn, -1)\n nn = self.conv(nn)\n nn = tf.squeeze(nn, -1)\n nn = tf.nn.sigmoid(nn)\n nn = nn[:,None,:]\n return inputs * nn\n\nclass CausalDWConv1D(tf.keras.layers.Layer):\n def __init__(self, \n kernel_size=17,\n dilation_rate=1,\n use_bias=False,\n depthwise_initializer='glorot_uniform',\n name='', **kwargs):\n super().__init__(name=name,**kwargs)\n self.causal_pad = tf.keras.layers.ZeroPadding1D((dilation_rate*(kernel_size-1),0),name=name + '_pad')\n self.dw_conv = tf.keras.layers.DepthwiseConv1D(\n kernel_size,\n strides=1,\n dilation_rate=dilation_rate,\n padding='valid',\n use_bias=use_bias,\n depthwise_initializer=depthwise_initializer,\n name=name + '_dwconv')\n self.supports_masking = True\n \n def call(self, inputs):\n x = self.causal_pad(inputs)\n x = self.dw_conv(x)\n return x\n\ndef Conv1DBlock(channel_size,\n kernel_size,\n dilation_rate=1,\n drop_rate=0.0,\n expand_ratio=2,\n se_ratio=0.25,\n activation='swish',\n name=None):\n '''\n efficient conv1d block, @hoyso48\n '''\n if name is None:\n name = str(tf.keras.backend.get_uid(\"mbblock\"))\n # Expansion phase\n def apply(inputs):\n channels_in = tf.keras.backend.int_shape(inputs)[-1]\n channels_expand = channels_in * expand_ratio\n\n skip = inputs\n\n x = tf.keras.layers.Dense(\n channels_expand,\n use_bias=True,\n activation=activation,\n name=name + '_expand_conv')(inputs)\n\n # Depthwise Convolution\n x = CausalDWConv1D(kernel_size,\n dilation_rate=dilation_rate,\n use_bias=False,\n name=name + '_dwconv')(x)\n\n x = tf.keras.layers.BatchNormalization(momentum=0.95, name=name + '_bn')(x)\n\n x = ECA()(x)\n\n x = tf.keras.layers.Dense(\n channel_size,\n use_bias=True,\n name=name + '_project_conv')(x)\n\n if drop_rate > 0:\n x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1), name=name + '_drop')(x)\n\n if (channels_in == channel_size):\n x = tf.keras.layers.add([x, skip], name=name + '_add')\n return x\n\n return apply\n\nclass MultiHeadSelfAttention(tf.keras.layers.Layer):\n def __init__(self, dim=256, num_heads=4, dropout=0, **kwargs):\n super().__init__(**kwargs)\n self.dim = dim\n self.scale = self.dim ** -0.5\n self.num_heads = num_heads\n self.qkv = tf.keras.layers.Dense(3 * dim, use_bias=False)\n self.drop1 = tf.keras.layers.Dropout(dropout)\n self.proj = tf.keras.layers.Dense(dim, use_bias=False)\n self.supports_masking = True\n\n def call(self, inputs, mask=None):\n qkv = self.qkv(inputs)\n qkv = tf.keras.layers.Permute((2, 1, 3))(tf.keras.layers.Reshape((-1, self.num_heads, self.dim * 3 // self.num_heads))(qkv))\n q, k, v = tf.split(qkv, [self.dim // self.num_heads] * 3, axis=-1)\n\n attn = tf.matmul(q, k, transpose_b=True) * self.scale\n\n if mask is not None:\n mask = mask[:, None, None, :]\n\n attn = tf.keras.layers.Softmax(axis=-1)(attn, mask=mask)\n attn = self.drop1(attn)\n\n x = attn @ v\n x = tf.keras.layers.Reshape((-1, self.dim))(tf.keras.layers.Permute((2, 1, 3))(x))\n x = self.proj(x)\n return x\n\n\ndef TransformerBlock(dim=256, num_heads=6, expand=4, attn_dropout=0.3, drop_rate=0.2, activation='swish'):\n def apply(inputs):\n x = inputs\n \n x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n x = MultiHeadSelfAttention(dim=dim,num_heads=num_heads,dropout=attn_dropout)(x)\n x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n x = tf.keras.layers.Dense(inputs.shape[-1], activation='relu')(x)\n x = tf.keras.layers.Add()([inputs, x])\n attn_out = x\n\n x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n x = tf.keras.layers.Dense(dim*expand, use_bias=False, activation=activation)(x)\n x = tf.keras.layers.Dense(inputs.shape[-1], use_bias=False)(x)\n x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n x = tf.keras.layers.Add()([attn_out, x])\n return x\n \n return apply\n\ndef positional_encoding(maxlen, num_hid):\n depth = num_hid/2\n positions = tf.range(maxlen, dtype = tf.float32)[..., tf.newaxis]\n depths = tf.range(depth, dtype = tf.float32)[np.newaxis, :]/depth\n angle_rates = tf.math.divide(1, tf.math.pow(tf.cast(10000, tf.float32), depths))\n angle_rads = tf.linalg.matmul(positions, angle_rates)\n pos_encoding = tf.concat(\n [tf.math.sin(angle_rads), tf.math.cos(angle_rads)],\n axis=-1)\n return pos_encoding","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:30.178929Z","iopub.execute_input":"2023-08-23T15:12:30.179316Z","iopub.status.idle":"2023-08-23T15:12:30.228165Z","shell.execute_reply.started":"2023-08-23T15:12:30.179288Z","shell.execute_reply":"2023-08-23T15:12:30.225854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def CTCLoss(labels, logits):\n label_length = tf.reduce_sum(tf.cast(labels != pad_token_idx, tf.int32), axis=-1)\n logit_length = tf.ones(tf.shape(logits)[0], dtype=tf.int32) * tf.shape(logits)[1]\n loss = tf.nn.ctc_loss(\n labels=labels,\n logits=logits,\n label_length=label_length,\n logit_length=logit_length,\n blank_index=pad_token_idx,\n logits_time_major=False\n )\n loss = tf.reduce_mean(loss)\n return loss","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:31.163104Z","iopub.execute_input":"2023-08-23T15:12:31.163534Z","iopub.status.idle":"2023-08-23T15:12:31.174615Z","shell.execute_reply.started":"2023-08-23T15:12:31.163501Z","shell.execute_reply":"2023-08-23T15:12:31.17322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_SHAPE = [128,276]","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:32.184236Z","iopub.execute_input":"2023-08-23T15:12:32.18476Z","iopub.status.idle":"2023-08-23T15:12:32.191787Z","shell.execute_reply.started":"2023-08-23T15:12:32.184725Z","shell.execute_reply":"2023-08-23T15:12:32.189698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(dim = 384):\n inp = tf.keras.Input(INPUT_SHAPE)\n x = inp\n \n x = tf.keras.layers.Masking(mask_value=0.0)(x)\n x = tf.keras.layers.Dense(dim, use_bias=False,name='stem_conv')(x) + positional_encoding(INPUT_SHAPE[0], dim)\n x = tf.keras.layers.BatchNormalization(momentum=0.95,name='stem_bn')(x)\n \n x = Conv1DBlock(dim, 17, drop_rate=0.4)(x)\n x = Conv1DBlock(dim, 11, drop_rate=0.4)(x)\n x = Conv1DBlock(dim, 7, drop_rate=0.4)(x)\n x = Conv1DBlock(dim, 7, drop_rate=0.4)(x)\n x = TransformerBlock(dim*2, num_heads=3, expand=2)(x)\n \n x = Conv1DBlock(dim, 17, drop_rate=0.4)(x)\n x = Conv1DBlock(dim, 11, drop_rate=0.4)(x)\n x = Conv1DBlock(dim, 5, drop_rate=0.4)(x)\n x = Conv1DBlock(dim, 3, drop_rate=0.4)(x)\n x = TransformerBlock(512, num_heads=4, expand=2)(x)\n \n x = Conv1DBlock(dim, 17, drop_rate=0.4)(x)\n x = Conv1DBlock(dim, 7, drop_rate=0.4)(x)\n x = Conv1DBlock(dim, 5, drop_rate=0.4)(x)\n x = Conv1DBlock(dim, 3, drop_rate=0.4)(x)\n x = TransformerBlock(256, num_heads=8, expand=2)(x)\n \n x = Conv1DBlock(dim, 11, drop_rate=0.45)(x)\n x = Conv1DBlock(dim, 7, drop_rate=0.45)(x)\n x = Conv1DBlock(dim, 5, drop_rate=0.45)(x)\n x = Conv1DBlock(dim, 3, drop_rate=0.45)(x)\n x = TransformerBlock(dim, expand=2)(x)\n \n x = tf.keras.layers.Dense(dim*2,activation='relu',name='top_conv')(x)\n x = tf.keras.layers.Dropout(0.4)(x)\n x = tf.keras.layers.Dense(len(char_to_num))(x)\n\n model = tf.keras.Model(inp, x)\n\n loss = CTCLoss\n \n # Adam Optimizer\n optimizer = tfa.optimizers.RectifiedAdam(sma_threshold=4)\n optimizer = tfa.optimizers.Lookahead(optimizer, sync_period=5)\n\n model.compile(loss=loss, optimizer=optimizer)\n\n return model\n\ntf.keras.backend.clear_session()\nmodel = get_model()\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:32.423405Z","iopub.execute_input":"2023-08-23T15:12:32.424845Z","iopub.status.idle":"2023-08-23T15:12:36.757892Z","shell.execute_reply.started":"2023-08-23T15:12:32.424784Z","shell.execute_reply":"2023-08-23T15:12:36.756606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\nwith STRATEGY.scope():\n model = get_model()\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:36.761261Z","iopub.execute_input":"2023-08-23T15:12:36.761791Z","iopub.status.idle":"2023-08-23T15:12:40.720839Z","shell.execute_reply.started":"2023-08-23T15:12:36.761745Z","shell.execute_reply":"2023-08-23T15:12:40.719527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k = model(foo[0])","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:40.722134Z","iopub.execute_input":"2023-08-23T15:12:40.722466Z","iopub.status.idle":"2023-08-23T15:12:43.301392Z","shell.execute_reply.started":"2023-08-23T15:12:40.722439Z","shell.execute_reply":"2023-08-23T15:12:43.300071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:43.303607Z","iopub.execute_input":"2023-08-23T15:12:43.304131Z","iopub.status.idle":"2023-08-23T15:12:43.766328Z","shell.execute_reply.started":"2023-08-23T15:12:43.304092Z","shell.execute_reply":"2023-08-23T15:12:43.764563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def num_to_char_fn(y):\n return [num_to_char.get(x, \"\") for x in y]\n\n@tf.function()\ndef decode_phrase(pred):\n x = tf.argmax(pred, axis=1)\n diff = tf.not_equal(x[:-1], x[1:])\n adjacent_indices = tf.where(diff)[:, 0]\n x = tf.gather(x, adjacent_indices)\n mask = x != pad_token_idx\n x = tf.boolean_mask(x, mask, axis=0)\n return x\n\n# A utility function to decode the output of the network\ndef decode_batch_predictions(pred):\n output_text = []\n for result in pred:\n result = \"\".join(num_to_char_fn(decode_phrase(result).numpy()))\n output_text.append(result)\n return output_text","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:04.296937Z","iopub.execute_input":"2023-08-23T15:12:04.297986Z","iopub.status.idle":"2023-08-23T15:12:04.310247Z","shell.execute_reply.started":"2023-08-23T15:12:04.297925Z","shell.execute_reply":"2023-08-23T15:12:04.308466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# A callback class to output a few transcriptions during training\nclass CallbackEval(tf.keras.callbacks.Callback):\n \"\"\"Displays a batch of outputs after every epoch.\"\"\"\n\n def __init__(self, dataset):\n super().__init__()\n self.dataset = dataset\n\n def on_epoch_end(self, epoch: int, logs=None):\n model.save_weights(\"model.h5\")\n predictions = []\n targets = []\n for batch in self.dataset:\n X, y = batch\n batch_predictions = model(X)\n batch_predictions = decode_batch_predictions(batch_predictions)\n predictions.extend(batch_predictions)\n for label in y:\n label = \"\".join(num_to_char_fn(label.numpy()))\n targets.append(label)\n print(\"-\" * 100)\n # for i in np.random.randint(0, len(predictions), 2):\n for i in range(32):\n print(f\"Target : {targets[i]}\")\n print(f\"Prediction: {predictions[i]}, len: {len(predictions[i])}\")\n print(\"-\" * 100)\n\n# Callback function to check transcription on the val set.\nvalidation_callback = CallbackEval(val_dataset.take(1))","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:04.311957Z","iopub.execute_input":"2023-08-23T15:12:04.312756Z","iopub.status.idle":"2023-08-23T15:12:04.347866Z","shell.execute_reply.started":"2023-08-23T15:12:04.312719Z","shell.execute_reply":"2023-08-23T15:12:04.345554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_EPOCHS = 70\nN_WARMUP_EPOCHS = 10\nLR_MAX = 1e-3\nWD_RATIO = 0.05\nWARMUP_METHOD = \"exp\"","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:04.349814Z","iopub.execute_input":"2023-08-23T15:12:04.350247Z","iopub.status.idle":"2023-08-23T15:12:04.361303Z","shell.execute_reply.started":"2023-08-23T15:12:04.350215Z","shell.execute_reply":"2023-08-23T15:12:04.35992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n \n if current_step < num_warmup_steps:\n if WARMUP_METHOD == 'log':\n return lr_max * 0.10 ** (num_warmup_steps - current_step)\n else:\n return lr_max * 2 ** -(num_warmup_steps - current_step)\n else:\n progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max\n \ndef plot_lr_schedule(lr_schedule, epochs):\n fig = plt.figure(figsize=(20, 10))\n plt.plot([None] + lr_schedule + [None])\n # X Labels\n x = np.arange(1, epochs + 1)\n x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n plt.xlim([1, epochs])\n plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n \n # Increase y-limit for better readability\n plt.ylim([0, max(lr_schedule) * 1.1])\n \n # Title\n schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n \n # Plot Learning Rates\n for x, val in enumerate(lr_schedule):\n if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n if x < len(lr_schedule) - 1:\n if lr_schedule[x - 1] < val:\n ha = 'right'\n else:\n ha = 'left'\n elif x == 0:\n ha = 'right'\n else:\n ha = 'left'\n plt.plot(x + 1, val, 'o', color='black');\n offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n \n plt.xlabel('Epoch', size=16, labelpad=5)\n plt.ylabel('Learning Rate', size=16, labelpad=5)\n plt.grid()\n plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=0)\n\n# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n def __init__(self, wd_ratio=WD_RATIO):\n self.step_counter = 0\n self.wd_ratio = wd_ratio\n \n def on_epoch_begin(self, epoch, logs=None):\n model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:04.363241Z","iopub.execute_input":"2023-08-23T15:12:04.363693Z","iopub.status.idle":"2023-08-23T15:12:05.331139Z","shell.execute_reply.started":"2023-08-23T15:12:04.363661Z","shell.execute_reply":"2023-08-23T15:12:05.329946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"history = model.fit(\n train_dataset,\n validation_data=val_dataset,\n epochs=N_EPOCHS,\n callbacks=[\n validation_callback,\n lr_callback,\n WeightDecayCallback(),\n ]\n)\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:12:05.332933Z","iopub.execute_input":"2023-08-23T15:12:05.333653Z","iopub.status.idle":"2023-08-23T15:12:05.340943Z","shell.execute_reply.started":"2023-08-23T15:12:05.333611Z","shell.execute_reply":"2023-08-23T15:12:05.339648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_path = [\n '/kaggle/input/ctc-trained-v-5/model_weights_on_epoch_30.h5',\n '/kaggle/input/ctc-trained-v-5/model_weights_on_epoch_40.h5',\n '/kaggle/input/ctc-trained-v-5/model_weights_on_epoch_50.h5',\n '/kaggle/input/ctc-trained-v-5/model_weights_on_epoch_80.h5',\n \n]","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:34:42.569258Z","iopub.execute_input":"2023-08-23T15:34:42.569682Z","iopub.status.idle":"2023-08-23T15:34:42.575862Z","shell.execute_reply.started":"2023-08-23T15:34:42.56965Z","shell.execute_reply":"2023-08-23T15:34:42.574739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [get_model() for _ in models_path]\nfor model,path in zip(models,models_path):\n model.load_weights(path)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:34:44.091416Z","iopub.execute_input":"2023-08-23T15:34:44.093063Z","iopub.status.idle":"2023-08-23T15:35:04.06648Z","shell.execute_reply.started":"2023-08-23T15:34:44.092997Z","shell.execute_reply":"2023-08-23T15:35:04.06526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"averaged_weights = []\nfor w_1,w_2,w_3,w_4 in zip(models[0].get_weights(), models[1].get_weights(), models[2].get_weights(), models[3].get_weights()):\n averaged_weights.append((w_1+w_2+w_3+w_4)/4)\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:35:07.849743Z","iopub.execute_input":"2023-08-23T15:35:07.850197Z","iopub.status.idle":"2023-08-23T15:35:08.340361Z","shell.execute_reply.started":"2023-08-23T15:35:07.850165Z","shell.execute_reply":"2023-08-23T15:35:08.33895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.set_weights(averaged_weights)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:35:10.138718Z","iopub.execute_input":"2023-08-23T15:35:10.139178Z","iopub.status.idle":"2023-08-23T15:35:10.230718Z","shell.execute_reply.started":"2023-08-23T15:35:10.139145Z","shell.execute_reply":"2023-08-23T15:35:10.228998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LOAD_WEIGHTS = True\nif LOAD_WEIGHTS:\n model.load_weights(\"/kaggle/input/ctc-trained-v-5/last_model.h5\")\n print(f'Successfully Loaded Pretrained Weights')","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:30:41.53891Z","iopub.execute_input":"2023-08-23T15:30:41.53944Z","iopub.status.idle":"2023-08-23T15:30:43.17215Z","shell.execute_reply.started":"2023-08-23T15:30:41.539394Z","shell.execute_reply":"2023-08-23T15:30:43.170387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TFLiteModel(tf.Module):\n def __init__(self, model):\n super(TFLiteModel, self).__init__()\n self.model = model\n \n @tf.function(input_signature=[tf.TensorSpec(shape=[None, len(SEL_COLS)], dtype=tf.float32, name='inputs')])\n def __call__(self, inputs, training=False):\n # Preprocess Data\n x = tf.cast(inputs, tf.float32)\n x = x[None]\n x = tf.cond(tf.shape(x)[1] == 0, lambda: tf.zeros((1, 1, len(SEL_COLS))), lambda: tf.identity(x))\n x = x[0]\n x = pre_process0(x)\n x = pre_process1(*x)\n x = tf.reshape(x, INPUT_SHAPE)\n x = x[None]\n x = self.model(x, training=False)\n x = x[0]\n x = decode_phrase(x)\n x = tf.cond(tf.shape(x)[0] == 0, lambda: tf.zeros(1, tf.int64), lambda: tf.identity(x))\n x = tf.one_hot(x, 59)\n return {'outputs': x}\n\ntflitemodel_base = TFLiteModel(model)\npred = tflitemodel_base(frames)[\"outputs\"]\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:35:14.938484Z","iopub.execute_input":"2023-08-23T15:35:14.938962Z","iopub.status.idle":"2023-08-23T15:35:18.259735Z","shell.execute_reply.started":"2023-08-23T15:35:14.938919Z","shell.execute_reply":"2023-08-23T15:35:18.258275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\".join(num_to_char_fn(decode_phrase(pred).numpy())),pred.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:35:18.26157Z","iopub.execute_input":"2023-08-23T15:35:18.26223Z","iopub.status.idle":"2023-08-23T15:35:18.274236Z","shell.execute_reply.started":"2023-08-23T15:35:18.262195Z","shell.execute_reply":"2023-08-23T15:35:18.272479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflitemodel_base)\nkeras_model_converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]#, tf.lite.OpsSet.SELECT_TF_OPS]\nkeras_model_converter.optimizations = [tf.lite.Optimize.DEFAULT]\nkeras_model_converter.target_spec.supported_types = [tf.float16]\ntflite_model = keras_model_converter.convert()\n\nwith open('model.tflite', 'wb') as f:\n f.write(tflite_model)\n \nwith open('inference_args.json', \"w\") as f:\n json.dump({\"selected_columns\" : SEL_COLS}, f)\n \n!zip submission.zip './model.tflite' './inference_args.json'","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:35:21.69854Z","iopub.execute_input":"2023-08-23T15:35:21.699028Z","iopub.status.idle":"2023-08-23T15:36:58.619938Z","shell.execute_reply.started":"2023-08-23T15:35:21.698994Z","shell.execute_reply":"2023-08-23T15:36:58.618189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open (\"inference_args.json\", \"r\") as f:\n SEL_COLS = json.load(f)[\"selected_columns\"]\n \ndef load_relevant_data_subset(pq_path):\n return pd.read_parquet(pq_path, columns=SEL_COLS)\n\ndef create_data_gen(file_ids, y_mul=1):\n def gen():\n for file_id in file_ids:\n pqfile = f\"{inpdir}/{file_id}.parquet\"\n seq_refs = df.loc[df.file_id == file_id]\n seqs = load_relevant_data_subset(pqfile)\n\n for seq_id in seq_refs.sequence_id:\n x = seqs.iloc[seqs.index == seq_id].to_numpy()\n y = str(df.loc[df.sequence_id == seq_id].phrase.iloc[0])\n \n r_nonan = np.sum(np.sum(np.isnan(x[:, RHAND_IDX_X]), axis = 1) == 0)\n l_nonan = np.sum(np.sum(np.isnan(x[:, LHAND_IDX_X]), axis = 1) == 0)\n no_nan = max(r_nonan, l_nonan)\n \n if y_mul*len(y)