{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Got my import\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\nimport tensorflow as tf\nimport tensorflow_addons as tfa\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit\nfrom leven import levenshtein\n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport time\nimport json\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\n# TQDM Progress Bar With Pandas Apply Function\ntqdm.pandas()\n\nprint(f'Tensorflow Version {tf.__version__}')\nprint(f'Python Version: {sys.version}')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-08T22:22:05.676957Z","iopub.execute_input":"2023-07-08T22:22:05.677316Z","iopub.status.idle":"2023-07-08T22:22:05.686308Z","shell.execute_reply.started":"2023-07-08T22:22:05.677287Z","shell.execute_reply":"2023-07-08T22:22:05.685406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#@title Set Seed\nfrom keras.utils import set_random_seed\nset_random_seed(20) # Setting the random seed to keras, numpy backend generator, and random.","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:22:07.435108Z","iopub.execute_input":"2023-07-08T22:22:07.435794Z","iopub.status.idle":"2023-07-08T22:22:07.440519Z","shell.execute_reply.started":"2023-07-08T22:22:07.435751Z","shell.execute_reply":"2023-07-08T22:22:07.439558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read Character to Ordinal Encoding Mapping\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    CHAR2ORD = json.load(json_file)\n    \n# Ordinal to Character Mapping\nORD2CHAR = {j:i for i,j in CHAR2ORD.items()}\n    \n# Character to Ordinal Encoding Mapping   \ndisplay(pd.Series(CHAR2ORD).to_frame('Ordinal Encoding'))","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:22:07.782081Z","iopub.execute_input":"2023-07-08T22:22:07.783157Z","iopub.status.idle":"2023-07-08T22:22:07.799949Z","shell.execute_reply.started":"2023-07-08T22:22:07.783111Z","shell.execute_reply":"2023-07-08T22:22:07.798934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_PHRASE_LENGTH = 128\nBATCH_SIZE = 64\nLR_MAX = 1e-3\nWD_RATIO = 0.05\nMAX_PHRASE_LENGTH = 31 + 1\nN_UNIQUE_CHARACTERS0 = len(CHAR2ORD)\nPAD_TOKEN = N_UNIQUE_CHARACTERS0 # This will be the position of Pad Token\nSOS_TOKEN = N_UNIQUE_CHARACTERS0 + 1 # This will be the position of the SOS Token\nEOS_TOKEN = N_UNIQUE_CHARACTERS0 + 2 # This will be the position of EOS Toekn\nN_UNIQUE_CHARACTERS = N_UNIQUE_CHARACTERS0 + 3\nN_TRAIN_SAMPLES = len(X_train)\nTRAIN_STEPS_PER_EPOCH = math.ceil(N_TRAIN_SAMPLES / BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:22:08.40226Z","iopub.execute_input":"2023-07-08T22:22:08.402608Z","iopub.status.idle":"2023-07-08T22:22:08.408598Z","shell.execute_reply.started":"2023-07-08T22:22:08.402578Z","shell.execute_reply":"2023-07-08T22:22:08.407718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/X.npy')\ny_train = np.load('/kaggle/input/aslfr-preprocessing-dataset/y.npy')[:,:MAX_PHRASE_LENGTH]\nN_TRAIN_SAMPLES = len(X_train)\nprint(f'X_train shape: {X_train.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:22:08.966173Z","iopub.execute_input":"2023-07-08T22:22:08.968028Z","iopub.status.idle":"2023-07-08T22:22:52.956671Z","shell.execute_reply.started":"2023-07-08T22:22:08.967985Z","shell.execute_reply":"2023-07-08T22:22:52.955066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_EXAMPLE_BATCH_SAMPLES = 1024\nN_EXAMPLE_BATCH_SAMPLES_SMALL = 32\n# Example Batch\nX_batch = {\n    'frames': np.copy(X_train[:N_EXAMPLE_BATCH_SAMPLES]),\n    'phrase': np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES]),\n#     'phrase_type': np.copy(y_phrase_type_train[:N_EXAMPLE_BATCH_SAMPLES]),\n}\ny_batch = np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES])\n# Small Example Batch\nX_batch_small = {\n    'frames': np.copy(X_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n    'phrase': np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n#     'phrase_type': np.copy(y_phrase_type_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL]),\n}\ny_batch_small = np.copy(y_train[:N_EXAMPLE_BATCH_SAMPLES_SMALL])","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:22:52.960851Z","iopub.execute_input":"2023-07-08T22:22:52.961214Z","iopub.status.idle":"2023-07-08T22:22:53.00372Z","shell.execute_reply.started":"2023-07-08T22:22:52.96118Z","shell.execute_reply":"2023-07-08T22:22:53.002788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INFERENCE_FILE_PATHS = pd.Series(\n        glob.glob('/kaggle/input/aslfr-preprocessing-dataset/train_landmark_subsets/*')\n    )\n\nprint(f'Found {len(INFERENCE_FILE_PATHS)} Inference Pickle Files')\n# Read First Parquet File\n# example_parquet_df = pd.read_parquet(train['file_path'][0])\nexample_parquet_df = pd.read_parquet(INFERENCE_FILE_PATHS[0])\n\n# Each parquet file contains 1000 recordings\nprint(f'# Unique Recording: {example_parquet_df.index.nunique()}')\n# Display DataFrame layout\ndisplay(example_parquet_df.head())","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:22:53.00497Z","iopub.execute_input":"2023-07-08T22:22:53.00526Z","iopub.status.idle":"2023-07-08T22:22:54.637884Z","shell.execute_reply.started":"2023-07-08T22:22:53.005236Z","shell.execute_reply":"2023-07-08T22:22:54.636961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get indices in original dataframe\ndef get_idxs(df, words_pos, words_neg=[], ret_names=True, idxs_pos=None):\n    \"\"\"\n    Returns column indices, or both column indices and names\n    Input: dataframe, body_name, words/letters to exclude, get names or not, exact positions to get \n    \"\"\"\n    idxs = []\n    names = []\n    for w in words_pos:\n        for col_idx, col in enumerate(example_parquet_df.columns):\n            # Exclude Non Landmark Columns\n            if col in ['frame']:\n                continue\n                \n            col_idx = int(col.split('_')[-1])\n            # Check if column name contains all words\n            if (w in col) and (idxs_pos is None or col_idx in idxs_pos) and all([w not in col for w in words_neg]):\n                idxs.append(col_idx)\n                names.append(col)\n    # Convert to Numpy arrays\n    idxs = np.array(idxs)\n    names = np.array(names)\n    # Returns either both column indices and names\n    if ret_names:\n        return idxs, names\n    # Or only columns indices\n    else:\n        return idxs","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:22:54.640714Z","iopub.execute_input":"2023-07-08T22:22:54.641412Z","iopub.status.idle":"2023-07-08T22:22:54.649211Z","shell.execute_reply.started":"2023-07-08T22:22:54.641378Z","shell.execute_reply":"2023-07-08T22:22:54.648556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lips Landmark Face Ids\n# #AM we could try adding more face landmarks.\nLIPS_LANDMARK_IDXS = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\n\n# Landmark Indices for Left/Right hand without z axis in raw data\nLEFT_HAND_IDXS0, LEFT_HAND_NAMES0 = get_idxs(example_parquet_df, ['left_hand'], ['z'])\nRIGHT_HAND_IDXS0, RIGHT_HAND_NAMES0 = get_idxs(example_parquet_df, ['right_hand'], ['z'])\nLIPS_IDXS0, LIPS_NAMES0 = get_idxs(example_parquet_df, ['face'], ['z'], idxs_pos=LIPS_LANDMARK_IDXS)\nCOLUMNS0 = np.concatenate((LEFT_HAND_NAMES0, RIGHT_HAND_NAMES0, LIPS_NAMES0))\nN_COLS0 = len(COLUMNS0)\n# Only X/Y axes are used\nN_DIMS0 = 2\n\nprint(f'N_COLS0: {N_COLS0}')","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:22:54.650962Z","iopub.execute_input":"2023-07-08T22:22:54.65168Z","iopub.status.idle":"2023-07-08T22:22:54.666814Z","shell.execute_reply.started":"2023-07-08T22:22:54.651619Z","shell.execute_reply":"2023-07-08T22:22:54.665797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Indices in processed data by axes with only dominant hand\nHAND_X_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'x' in name]\n    ).squeeze()\nHAND_Y_IDXS = np.array(\n        [idx for idx, name in enumerate(LEFT_HAND_NAMES0) if 'y' in name]\n    ).squeeze()\n# Names in processed data by axes\nHAND_X_NAMES = LEFT_HAND_NAMES0[HAND_X_IDXS]\nHAND_Y_NAMES = LEFT_HAND_NAMES0[HAND_Y_IDXS]\nHAND_X_NAMES.shape, HAND_Y_NAMES.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:22:54.668353Z","iopub.execute_input":"2023-07-08T22:22:54.668999Z","iopub.status.idle":"2023-07-08T22:22:54.679859Z","shell.execute_reply.started":"2023-07-08T22:22:54.668969Z","shell.execute_reply":"2023-07-08T22:22:54.678964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HAND_X_NAMES","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:22:54.681066Z","iopub.execute_input":"2023-07-08T22:22:54.681591Z","iopub.status.idle":"2023-07-08T22:22:54.693568Z","shell.execute_reply.started":"2023-07-08T22:22:54.681561Z","shell.execute_reply":"2023-07-08T22:22:54.692796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:21:43.860991Z","iopub.execute_input":"2023-07-08T14:21:43.861642Z","iopub.status.idle":"2023-07-08T14:21:43.872507Z","shell.execute_reply.started":"2023-07-08T14:21:43.861604Z","shell.execute_reply":"2023-07-08T14:21:43.871478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_dataset(X, y, batch_size=64):\n    sample_idxs = np.arange(len(X))\n    while True:\n        # Get random indices\n        random_sample_idxs = np.random.choice(sample_idxs, batch_size)\n        \n        # Only frames are required as model input\n        inputs = X[random_sample_idxs]\n        outputs = y[random_sample_idxs]\n        \n        yield inputs, outputs","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:23:17.569393Z","iopub.execute_input":"2023-07-08T22:23:17.56978Z","iopub.status.idle":"2023-07-08T22:23:17.576046Z","shell.execute_reply.started":"2023-07-08T22:23:17.569727Z","shell.execute_reply":"2023-07-08T22:23:17.575111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_dataset= get_train_dataset(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:23:21.847843Z","iopub.execute_input":"2023-07-08T22:23:21.848214Z","iopub.status.idle":"2023-07-08T22:23:21.852977Z","shell.execute_reply.started":"2023-07-08T22:23:21.848183Z","shell.execute_reply":"2023-07-08T22:23:21.852064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import callbacks\ncallback = callbacks.EarlyStopping(monitor='train_loss', patience=3)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:23:24.093906Z","iopub.execute_input":"2023-07-08T22:23:24.094279Z","iopub.status.idle":"2023-07-08T22:23:24.098911Z","shell.execute_reply.started":"2023-07-08T22:23:24.094248Z","shell.execute_reply":"2023-07-08T22:23:24.097954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import TimeDistributed\nfrom keras import Input\nfrom tensorflow.keras.applications.resnet50 import preprocess_input # Specificly for ResNet50\nfrom keras.applications import ResNet50\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Flatten, GlobalAveragePooling2D, LSTM\nfrom keras.initializers import HeNormal\nfrom keras.layers import Add, Input, Flatten\nfrom keras.optimizers import Adam\nfrom keras.utils import plot_model\n","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:23:24.773198Z","iopub.execute_input":"2023-07-08T22:23:24.774094Z","iopub.status.idle":"2023-07-08T22:23:24.7836Z","shell.execute_reply.started":"2023-07-08T22:23:24.774051Z","shell.execute_reply":"2023-07-08T22:23:24.782513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from keras.models import Model\n# from keras.layers import TimeDistributed\n# from keras import Input\n# from tensorflow.keras.applications.resnet50 import preprocess_input # Specificly for ResNet50\n# from keras.applications import ResNet50\n# from keras.models import Sequential\n# from keras.layers import Dense, Flatten, GlobalAveragePooling2D, LSTM\n# from keras.initializers import HeNormal\n# from keras.layers import Add, Input, Flatten\n# from keras.optimizers import Adam\n\n# optimizer = Adam(learning_rate=1e-4, clipnorm=0.5)\n# initializer = HeNormal(seed=20) # Trying to see what will happen with different initializer\n\n# # Defining the CNN model\n# def cnn_model():\n#     resnet = ResNet50(include_top=False, pooling='avg')\n#     resnet.trainable = False # No need to train ResNet50 \n#     model = Sequential()\n#     model.add(resnet)\n#     model.add(Dense(128, activation='relu'))\n#     return model\n\n\n# # Input for the combined model\n# combined_input = Input(shape=(128, 164, 164, 3)) # assuming each frame is 128x128 pixels with 3 color channels\n\n# # Time distributed CNN model\n# cnn = cnn_model()\n# time_distributed_cnn = TimeDistributed(cnn)(combined_input)\n\n# # LSTM model\n# lstm_1 = LSTM(128, name='lstm_1', return_sequences=True)(time_distributed_cnn)\n# lstm_2 = LSTM(128, name='lstm_2', return_sequences=True)(lstm_1)\n# lstm_3 = LSTM(128, name='lstm_3', return_sequences=True)(lstm_2)\n# lstm_4 = LSTM(128, name='lstm_4', return_sequences=True)(lstm_3)\n\n# # Extract the desired sequence length\n# sliced_output = lstm_4[:, :31, :]  # Slice the output to keep only the first 31 time steps\n\n# # Dense layers\n# dense_match = Dense(128, name='dense_match')(Flatten()(lstm_1))\n# dense_1 = Dense(128, name='dense_1', activation='relu')(sliced_output)\n# dense_1 = Add()([dense_1, dense_match])\n# output = Dense(62, name='output', activation='softmax')(dense_1)\n\n# # Combine the models\n# combined_model = Model(combined_input, output)\n# combined_model.compile(\n#     loss=loss,\n#     optimizer=optimizer,\n#     metrics=metrics,\n#     loss_weights=loss_weights,\n# )\n","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:23:29.015547Z","iopub.execute_input":"2023-07-08T22:23:29.015919Z","iopub.status.idle":"2023-07-08T22:23:29.02333Z","shell.execute_reply.started":"2023-07-08T22:23:29.015888Z","shell.execute_reply":"2023-07-08T22:23:29.02199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting model\nplot_model(combined_model , \"simple_model.png\", show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:23:29.336038Z","iopub.execute_input":"2023-07-08T22:23:29.336393Z","iopub.status.idle":"2023-07-08T22:23:29.377352Z","shell.execute_reply.started":"2023-07-08T22:23:29.336365Z","shell.execute_reply":"2023-07-08T22:23:29.375925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:23:31.374063Z","iopub.execute_input":"2023-07-08T22:23:31.374455Z","iopub.status.idle":"2023-07-08T22:23:31.418518Z","shell.execute_reply.started":"2023-07-08T22:23:31.374424Z","shell.execute_reply":"2023-07-08T22:23:31.41697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import LSTM, Dense, Lambda\n\n# Assume we're working with single-feature sequences.\nn_features = 1\n\n# Create the model.\nmodel = Sequential()\n\n# Add LSTM layer.\nmodel.add(LSTM(50, activation='relu', input_shape=(10, n_features), return_sequences=True))\n\n# Add a lambda layer to keep only the last 3 outputs of the LSTM.\nmodel.add(Lambda(lambda x: x[:, -3:, :]))\n\n# For each of the 3 time steps, output a single value.\nmodel.add(TimeDistributed(Dense(1)))\n\n# Compile the model.\nmodel.compile(optimizer='adam', loss='mse')\n\n# Now the model will take inputs of shape (batch_size, 10, 1) \n# and output sequences of shape (batch_size, 3, 1).\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:24:18.911582Z","iopub.execute_input":"2023-07-08T22:24:18.91195Z","iopub.status.idle":"2023-07-08T22:24:19.034956Z","shell.execute_reply.started":"2023-07-08T22:24:18.91192Z","shell.execute_reply":"2023-07-08T22:24:19.034123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model , \"simple_model.png\", show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:24:19.172171Z","iopub.execute_input":"2023-07-08T22:24:19.172498Z","iopub.status.idle":"2023-07-08T22:24:19.234628Z","shell.execute_reply.started":"2023-07-08T22:24:19.172462Z","shell.execute_reply":"2023-07-08T22:24:19.233708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LSTM Model only:","metadata":{}},{"cell_type":"code","source":"def scce_with_ls(y_true, y_pred):\n    # Filter Pad Tokens\n    idxs = tf.where(y_true != PAD_TOKEN)\n    y_true = tf.gather_nd(y_true, idxs)\n    y_pred = tf.gather_nd(y_pred, idxs)\n    # One Hot Encode Sparsely Encoded Target Sign\n    y_true = tf.cast(y_true, tf.int32)\n    y_true = tf.one_hot(y_true, N_UNIQUE_CHARACTERS, axis=1)\n    # Categorical Crossentropy with native label smoothing support\n    loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, label_smoothing=0.25, from_logits=True)\n    loss = tf.math.reduce_mean(loss)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:24:21.049354Z","iopub.execute_input":"2023-07-08T22:24:21.049705Z","iopub.status.idle":"2023-07-08T22:24:21.056328Z","shell.execute_reply.started":"2023-07-08T22:24:21.049677Z","shell.execute_reply":"2023-07-08T22:24:21.055189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = scce_with_ls","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:24:21.733349Z","iopub.execute_input":"2023-07-08T22:24:21.73404Z","iopub.status.idle":"2023-07-08T22:24:21.73811Z","shell.execute_reply.started":"2023-07-08T22:24:21.734008Z","shell.execute_reply":"2023-07-08T22:24:21.737185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = tfa.optimizers.RectifiedAdam(sma_threshold=4)\noptimizer = tfa.optimizers.Lookahead(optimizer, sync_period=5)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:24:21.988333Z","iopub.execute_input":"2023-07-08T22:24:21.990353Z","iopub.status.idle":"2023-07-08T22:24:21.995969Z","shell.execute_reply.started":"2023-07-08T22:24:21.990321Z","shell.execute_reply":"2023-07-08T22:24:21.995004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:24:22.638427Z","iopub.execute_input":"2023-07-08T22:24:22.638814Z","iopub.status.idle":"2023-07-08T22:24:22.647026Z","shell.execute_reply.started":"2023-07-08T22:24:22.638782Z","shell.execute_reply":"2023-07-08T22:24:22.646028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train[0]","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:36:49.187219Z","iopub.execute_input":"2023-07-08T22:36:49.187604Z","iopub.status.idle":"2023-07-08T22:36:49.195356Z","shell.execute_reply.started":"2023-07-08T22:36:49.187572Z","shell.execute_reply":"2023-07-08T22:36:49.194197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TopK accuracy for multi dimensional output\nclass TopKAccuracy(tf.keras.metrics.Metric):\n    def __init__(self, k, **kwargs):\n        super(TopKAccuracy, self).__init__(name=f'top{k}acc', **kwargs)\n        self.top_k_acc = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=k)\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        y_true = tf.reshape(y_true, [-1])\n        y_pred = tf.reshape(y_pred, [-1, N_UNIQUE_CHARACTERS])\n        character_idxs = tf.where(y_true < N_UNIQUE_CHARACTERS0)\n        y_true = tf.gather(y_true, character_idxs, axis=0)\n        y_pred = tf.gather(y_pred, character_idxs, axis=0)\n        self.top_k_acc.update_state(y_true, y_pred)\n\n    def result(self):\n        return self.top_k_acc.result()\n    \n    def reset_state(self):\n        self.top_k_acc.reset_state()","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:24:24.21744Z","iopub.execute_input":"2023-07-08T22:24:24.217851Z","iopub.status.idle":"2023-07-08T22:24:24.228249Z","shell.execute_reply.started":"2023-07-08T22:24:24.217818Z","shell.execute_reply":"2023-07-08T22:24:24.227097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" metrics = [\n        TopKAccuracy(1),\n        TopKAccuracy(5),\n    ]","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:24:25.271342Z","iopub.execute_input":"2023-07-08T22:24:25.271705Z","iopub.status.idle":"2023-07-08T22:24:25.285679Z","shell.execute_reply.started":"2023-07-08T22:24:25.271675Z","shell.execute_reply":"2023-07-08T22:24:25.28478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Initial Loss Weights All Set To 1\nloss_weights = np.ones(N_UNIQUE_CHARACTERS, dtype=np.float32)\n# Set Loss Weight Of Pad Token To 0\nloss_weights[PAD_TOKEN] = 0","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:24:25.846178Z","iopub.execute_input":"2023-07-08T22:24:25.846499Z","iopub.status.idle":"2023-07-08T22:24:25.852392Z","shell.execute_reply.started":"2023-07-08T22:24:25.846472Z","shell.execute_reply":"2023-07-08T22:24:25.850388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# input_layer = Input(shape=(128, 164), name='input')\n# lstm_1 = LSTM(128, name='lstm_1', return_sequences=True)(input_layer)\n# lstm_2 = LSTM(128, name='lstm_2', return_sequences=True)(lstm_1)\n# lstm_3 = LSTM(128, name='lstm_3', return_sequences=True)(lstm_2)\n# lstm_4 = LSTM(128, name='lstm_4', return_sequences=True)(lstm_3)\n\n\n# # Extract the desired sequence length\n# sliced_output = lstm_4[:, :32, :]  # Slice the output to keep only the first 31 time steps\n\n# # Dense layers\n# dense_match = Dense(128, name='dense_match')(Flatten()(lstm_1[:,:32, :]))\n# dense_1 = Dense(128, name='dense_1', activation='relu')(sliced_output)\n# dense_1 = Add()([dense_1, dense_match])\n# output = Dense(62, name='output', activation='softmax')(dense_1)\n\n# model_lstm = Model(input_layer, output)\n# model_lstm.compile(\n#     loss=loss,\n#     optimizer=optimizer,\n#     metrics=metrics,\n#     loss_weights=loss_weights,\n# )","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:32:48.623743Z","iopub.execute_input":"2023-07-08T14:32:48.624126Z","iopub.status.idle":"2023-07-08T14:32:49.69505Z","shell.execute_reply.started":"2023-07-08T14:32:48.624098Z","shell.execute_reply":"2023-07-08T14:32:49.694071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_lstm.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:32:49.697061Z","iopub.execute_input":"2023-07-08T14:32:49.697419Z","iopub.status.idle":"2023-07-08T14:32:49.730467Z","shell.execute_reply.started":"2023-07-08T14:32:49.697385Z","shell.execute_reply":"2023-07-08T14:32:49.729728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import plot_model\n\nplot_model(model_lstm , \"LSTM_model.png\", show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:32:51.202702Z","iopub.execute_input":"2023-07-08T14:32:51.203446Z","iopub.status.idle":"2023-07-08T14:32:51.350632Z","shell.execute_reply.started":"2023-07-08T14:32:51.203409Z","shell.execute_reply":"2023-07-08T14:32:51.349751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model above makes no sense. ","metadata":{}},{"cell_type":"code","source":"train_dataset= get_train_dataset(X_train, y_train)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:31:30.011001Z","iopub.execute_input":"2023-07-08T14:31:30.011388Z","iopub.status.idle":"2023-07-08T14:31:30.016419Z","shell.execute_reply.started":"2023-07-08T14:31:30.011358Z","shell.execute_reply":"2023-07-08T14:31:30.014782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 1000","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:35:06.424313Z","iopub.execute_input":"2023-07-08T14:35:06.424709Z","iopub.status.idle":"2023-07-08T14:35:06.429534Z","shell.execute_reply.started":"2023-07-08T14:35:06.424672Z","shell.execute_reply":"2023-07-08T14:35:06.428377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# changed everything to Timedistributed\ninput_layer = Input(shape=(128, 164), name='input')\nlstm_1 = LSTM(128, name='lstm_1', return_sequences=True)(input_layer)\nlstm_2 = LSTM(128, name='lstm_2', return_sequences=True)(lstm_1)\nlstm_3 = LSTM(128, name='lstm_3', return_sequences=True)(lstm_2)\nlstm_4 = LSTM(128, name='lstm_4', return_sequences=True)(Add()([lstm_1, lstm_3]))\n\n\n# Extract the desired sequence length\nsliced_output = lstm_4[:, :32, :]  # Slice the output to keep only the first 31 time steps\n\n# Dense layers\ndense_1 = TimeDistributed(Dense(128, name='dense_1', activation='relu'))(sliced_output)\noutput = TimeDistributed(Dense(62, name='output', activation='softmax'))(dense_1)\n\nmodel_lstm = Model(input_layer, output)\nmodel_lstm.compile(\n    loss=loss,\n    optimizer=optimizer,\n    metrics=metrics,\n    loss_weights=loss_weights,\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:25:49.432228Z","iopub.execute_input":"2023-07-08T22:25:49.432694Z","iopub.status.idle":"2023-07-08T22:25:50.515913Z","shell.execute_reply.started":"2023-07-08T22:25:49.432652Z","shell.execute_reply":"2023-07-08T22:25:50.514987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lstm.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:25:51.854517Z","iopub.execute_input":"2023-07-08T22:25:51.854905Z","iopub.status.idle":"2023-07-08T22:25:51.891131Z","shell.execute_reply.started":"2023-07-08T22:25:51.854874Z","shell.execute_reply":"2023-07-08T22:25:51.890434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model_lstm , \"simple_model.png\", show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:25:52.817289Z","iopub.execute_input":"2023-07-08T22:25:52.81764Z","iopub.status.idle":"2023-07-08T22:25:52.922895Z","shell.execute_reply.started":"2023-07-08T22:25:52.817605Z","shell.execute_reply":"2023-07-08T22:25:52.921986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model_lstm.fit(x=train_dataset, steps_per_epoch=TRAIN_STEPS_PER_EPOCH, epochs=100, verbose=True, callbacks=[callback])\n","metadata":{"execution":{"iopub.status.busy":"2023-07-08T22:38:15.827214Z","iopub.execute_input":"2023-07-08T22:38:15.827593Z","iopub.status.idle":"2023-07-08T23:01:02.990226Z","shell.execute_reply.started":"2023-07-08T22:38:15.827563Z","shell.execute_reply":"2023-07-08T23:01:02.988601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lstm.save('LSTM_skip.h5')","metadata":{"execution":{"iopub.status.busy":"2023-07-08T23:01:02.991361Z","iopub.status.idle":"2023-07-08T23:01:02.992504Z","shell.execute_reply.started":"2023-07-08T23:01:02.992257Z","shell.execute_reply":"2023-07-08T23:01:02.99228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred=model_lstm.predict(X_train[1:2])","metadata":{"execution":{"iopub.status.busy":"2023-07-08T15:02:16.887357Z","iopub.execute_input":"2023-07-08T15:02:16.887971Z","iopub.status.idle":"2023-07-08T15:02:18.332693Z","shell.execute_reply.started":"2023-07-08T15:02:16.887926Z","shell.execute_reply":"2023-07-08T15:02:18.331475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_c =pred.argmax(axis=2)[0]\nfor i in pred_c:\n    print(ORD2CHAR[i])","metadata":{"execution":{"iopub.status.busy":"2023-07-08T15:01:18.311621Z","iopub.execute_input":"2023-07-08T15:01:18.311947Z","iopub.status.idle":"2023-07-08T15:01:18.321007Z","shell.execute_reply.started":"2023-07-08T15:01:18.31192Z","shell.execute_reply":"2023-07-08T15:01:18.320053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in y_train[1]:\n    print(ORD2CHAR[i][])","metadata":{"execution":{"iopub.status.busy":"2023-07-08T15:01:18.323738Z","iopub.execute_input":"2023-07-08T15:01:18.324325Z","iopub.status.idle":"2023-07-08T15:01:18.369272Z","shell.execute_reply.started":"2023-07-08T15:01:18.324292Z","shell.execute_reply":"2023-07-08T15:01:18.367956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:42:35.090583Z","iopub.execute_input":"2023-07-08T14:42:35.091003Z","iopub.status.idle":"2023-07-08T14:42:35.096088Z","shell.execute_reply.started":"2023-07-08T14:42:35.090968Z","shell.execute_reply":"2023-07-08T14:42:35.094825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:32:06.594283Z","iopub.execute_input":"2023-07-08T14:32:06.59464Z","iopub.status.idle":"2023-07-08T14:32:06.599076Z","shell.execute_reply.started":"2023-07-08T14:32:06.59461Z","shell.execute_reply":"2023-07-08T14:32:06.598133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model_lstm.fit(x=train_dataset, steps_per_epoch=TRAIN_STEPS_PER_EPOCH, epochs=EPOCHS, verbose=True, callbacks=[callback])\n","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:32:06.934319Z","iopub.execute_input":"2023-07-08T14:32:06.934668Z","iopub.status.idle":"2023-07-08T14:32:48.473408Z","shell.execute_reply.started":"2023-07-08T14:32:06.934638Z","shell.execute_reply":"2023-07-08T14:32:48.472438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1 LSTM layer\ninput_layer = Input(shape=(128, 164), name='input')\nlstm_1 = LSTM(128, name='lstm_1', return_sequences=True)(input_layer)\noutput = TimeDistributed(Dense(62, name='output', activation='softmax'))(lstm_1)\n\nmodel_lstm_simple = Model(input_layer, output)\nmodel_lstm_simple.compile(\n    loss=loss,\n    optimizer=optimizer,\n    metrics=metrics,\n    loss_weights=loss_weights,\n)\nmodel_lstm_simple.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:22:43.364732Z","iopub.execute_input":"2023-07-08T14:22:43.36539Z","iopub.status.idle":"2023-07-08T14:22:43.767585Z","shell.execute_reply.started":"2023-07-08T14:22:43.365363Z","shell.execute_reply":"2023-07-08T14:22:43.76685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model_lstm_simple)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:23:04.994001Z","iopub.execute_input":"2023-07-08T14:23:04.994389Z","iopub.status.idle":"2023-07-08T14:23:05.172976Z","shell.execute_reply.started":"2023-07-08T14:23:04.994357Z","shell.execute_reply":"2023-07-08T14:23:05.171864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model_lstm_simple.fit(x=train_dataset,  epochs=EPOCHS, verbose=True, callbacks=[callback])\n","metadata":{"execution":{"iopub.status.busy":"2023-07-08T14:23:11.11524Z","iopub.execute_input":"2023-07-08T14:23:11.11559Z","iopub.status.idle":"2023-07-08T14:28:10.677236Z","shell.execute_reply.started":"2023-07-08T14:23:11.115562Z","shell.execute_reply":"2023-07-08T14:28:10.675735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}