{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":52950,"databundleVersionId":5973250,"sourceType":"competition"}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n# Google - American Sign Language Fingerspelling Recognition\n
\nCompetition Link: https://www.kaggle.com/competitions/asl-fingerspelling/data\n
\nGenerating text from American Sign Language (ASL) using keypoint data and an encoder-decoder architecture is a promising leap in accessibility technology. ASL is a vital means of communication for the Deaf community, but its understanding is not universal. This innovative approach leverages computer vision to capture ASL gestures as keypoint data and translates it into written text.The encoder processes the keypoints, extracting meaningful information, while the decoder converts this data into a coherent text representation. This technology holds immense potential for bridging communication gaps between Deaf and hearing individuals, making information and conversations more inclusive. It's a significant step toward a more inclusive and connected world, demonstrating the power of AI to break down barriers.\n
","metadata":{}},{"cell_type":"markdown","source":"# Sample Hand Landmarks Example\n","metadata":{}},{"cell_type":"markdown","source":"# Encoder-Decoder Architecture With Bidirectional-LSTM","metadata":{}},{"cell_type":"markdown","source":"## Import Dependencies","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\npd.set_option('display.max_columns',300)\nimport json\nimport tensorflow as tf\nfrom tensorflow.keras.layers import LSTM, Bidirectional, Dropout, Embedding, Dense, Input, Lambda, Conv1D, MaxPool1D\nfrom tensorflow.keras.models import Model\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:16.471125Z","iopub.execute_input":"2024-01-10T10:27:16.471417Z","iopub.status.idle":"2024-01-10T10:27:26.746951Z","shell.execute_reply.started":"2024-01-10T10:27:16.471392Z","shell.execute_reply":"2024-01-10T10:27:26.745878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read Train Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/asl-fingerspelling/train.csv\")\ndf","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:26.749075Z","iopub.execute_input":"2024-01-10T10:27:26.74979Z","iopub.status.idle":"2024-01-10T10:27:26.920049Z","shell.execute_reply.started":"2024-01-10T10:27:26.749748Z","shell.execute_reply":"2024-01-10T10:27:26.919131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Filter Out Hand Landmarks","metadata":{}},{"cell_type":"code","source":"# ! pip install fastparquet pyarrow","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:26.921321Z","iopub.execute_input":"2024-01-10T10:27:26.921691Z","iopub.status.idle":"2024-01-10T10:27:26.926174Z","shell.execute_reply.started":"2024-01-10T10:27:26.921655Z","shell.execute_reply":"2024-01-10T10:27:26.925079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landm = pd.read_parquet(\"/kaggle/input/asl-fingerspelling/train_landmarks/1019715464.parquet\")\nlandm","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:26.928256Z","iopub.execute_input":"2024-01-10T10:27:26.928537Z","iopub.status.idle":"2024-01-10T10:27:46.189112Z","shell.execute_reply.started":"2024-01-10T10:27:26.928513Z","shell.execute_reply":"2024-01-10T10:27:46.188129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"handcolumns = [c for c in landm.columns if \"hand\" in c]","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.190739Z","iopub.execute_input":"2024-01-10T10:27:46.191123Z","iopub.status.idle":"2024-01-10T10:27:46.197217Z","shell.execute_reply.started":"2024-01-10T10:27:46.191067Z","shell.execute_reply":"2024-01-10T10:27:46.196033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del landm","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.198629Z","iopub.execute_input":"2024-01-10T10:27:46.198966Z","iopub.status.idle":"2024-01-10T10:27:46.211198Z","shell.execute_reply.started":"2024-01-10T10:27:46.198934Z","shell.execute_reply":"2024-01-10T10:27:46.210283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Characterset","metadata":{}},{"cell_type":"code","source":"with open(\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n charmap = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.212393Z","iopub.execute_input":"2024-01-10T10:27:46.212783Z","iopub.status.idle":"2024-01-10T10:27:46.226614Z","shell.execute_reply.started":"2024-01-10T10:27:46.212757Z","shell.execute_reply":"2024-01-10T10:27:46.225844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"charmap = {k:v+4 for k,v in charmap.items()}\ncharmap[\"[START]\"]=1\ncharmap['[PAD]']=0\ncharmap['[UNK]']=2\ncharmap['[END]']=3","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.227738Z","iopub.execute_input":"2024-01-10T10:27:46.228064Z","iopub.status.idle":"2024-01-10T10:27:46.233728Z","shell.execute_reply.started":"2024-01-10T10:27:46.228034Z","shell.execute_reply":"2024-01-10T10:27:46.232773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split Dataset","metadata":{}},{"cell_type":"code","source":"train_df, test_df = train_test_split(df, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.234897Z","iopub.execute_input":"2024-01-10T10:27:46.235278Z","iopub.status.idle":"2024-01-10T10:27:46.267446Z","shell.execute_reply.started":"2024-01-10T10:27:46.235245Z","shell.execute_reply":"2024-01-10T10:27:46.266721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape, test_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.270455Z","iopub.execute_input":"2024-01-10T10:27:46.270727Z","iopub.status.idle":"2024-01-10T10:27:46.275937Z","shell.execute_reply.started":"2024-01-10T10:27:46.270703Z","shell.execute_reply":"2024-01-10T10:27:46.275136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.276914Z","iopub.execute_input":"2024-01-10T10:27:46.27721Z","iopub.status.idle":"2024-01-10T10:27:46.2849Z","shell.execute_reply.started":"2024-01-10T10:27:46.277188Z","shell.execute_reply":"2024-01-10T10:27:46.283968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Dataset Generator","metadata":{}},{"cell_type":"code","source":"import re\n\ndef is_numeric_and_punctuation_string(input_string):\n # Define a regular expression pattern for numbers and punctuation marks\n pattern = r'^[0-9.,;:\"\\'?!()\\[\\]{}\\-+*/%&@#<>=]*$'\n\n # Use re.match to check if the entire string matches the pattern\n match = re.match(pattern, input_string)\n\n # If there is a match, the string contains only numbers and punctuation marks\n return bool(match)\n\n# Example usage:\ninput_str = \"123,456.789!\"\nresult = is_numeric_and_punctuation_string(input_str)\nprint(result)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.286004Z","iopub.execute_input":"2024-01-10T10:27:46.286307Z","iopub.status.idle":"2024-01-10T10:27:46.295985Z","shell.execute_reply.started":"2024-01-10T10:27:46.286284Z","shell.execute_reply":"2024-01-10T10:27:46.295042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_manager = dict()","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.297267Z","iopub.execute_input":"2024-01-10T10:27:46.29753Z","iopub.status.idle":"2024-01-10T10:27:46.305821Z","shell.execute_reply.started":"2024-01-10T10:27:46.297508Z","shell.execute_reply":"2024-01-10T10:27:46.305054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generator(train, handcolumns, charmaps, max_frame, max_char):\n train = train.sort_values(by='file_id')\n while True:\n for fileid in train[\"file_id\"].unique():\n if not fileid in file_manager.keys():\n path = f\"/kaggle/input/asl-fingerspelling/train_landmarks/{fileid}.parquet\"\n land_marks = pd.read_parquet(path)[handcolumns]\n for col in handcolumns: \n land_marks[col] = land_marks[col].astype('float16', errors='ignore')\n file_manager[fileid]=land_marks\n else:\n land_marks = file_manager[fileid]\n land_marks.dropna(how='all', axis=0, inplace=True)\n land_marks.fillna(0.0, inplace=True)\n frameCount = pd.DataFrame(land_marks.index.value_counts())\n frameCount[\"id\"] = frameCount.index\n frameCount.columns = [\"frameCount\", \"id\"]\n this_file_seqs = train[train[\"file_id\"]==fileid]\n \n d = pd.merge(this_file_seqs[[\"sequence_id\", \"phrase\"]], frameCount, how='inner', left_on=\"sequence_id\", right_on=\"id\")\n d[\"len\"] = d[\"phrase\"].apply(len)\n d[\"avgF/Char\"] = d[\"frameCount\"]/d[\"len\"]\n f = d[(d[\"avgF/Char\"]>5) & (d[\"avgF/Char\"]<=15)]\n for seqId, phrase in f[[\"sequence_id\", \"phrase\"]].values:\n if is_numeric_and_punctuation_string(phrase) or phrase.startswith(\"http\") or phrase.startswith(\"www\"):\n continue\n allframes = land_marks.loc[seqId].values[:max_frame] # limit to max number of frames\n pads =np.zeros(shape=(max_frame-len(allframes), len(handcolumns)))\n if pads.shape[0]!=0:\n padded_frames = np.row_stack((pads, allframes))\n else:\n padded_frames = allframes\n phrase = phrase[:min((max_char-1), len(phrase))] # limit max number of characters\n target_in = [1] + [charmaps[c] for c in phrase] + [0.0]*(max_char - len(phrase)-1) # [START] + [index of chars] + post pad with 0\n target_out = [charmaps[c] for c in phrase] + [3] + [0.0]*(max_char - len(phrase)-1) # [index of chars] + [END] + post pad with 0\n yield (tf.cast(np.array(padded_frames), dtype=tf.float32), np.array(target_in, dtype='int8')), np.array(target_out, dtype='int8')","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.307159Z","iopub.execute_input":"2024-01-10T10:27:46.307654Z","iopub.status.idle":"2024-01-10T10:27:46.323053Z","shell.execute_reply.started":"2024-01-10T10:27:46.307623Z","shell.execute_reply":"2024-01-10T10:27:46.32213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"http\".startswith","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.324326Z","iopub.execute_input":"2024-01-10T10:27:46.324641Z","iopub.status.idle":"2024-01-10T10:27:46.336108Z","shell.execute_reply.started":"2024-01-10T10:27:46.324618Z","shell.execute_reply":"2024-01-10T10:27:46.335283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_frame = 300\nmax_char = 75","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.337069Z","iopub.execute_input":"2024-01-10T10:27:46.337331Z","iopub.status.idle":"2024-01-10T10:27:46.346123Z","shell.execute_reply.started":"2024-01-10T10:27:46.33731Z","shell.execute_reply":"2024-01-10T10:27:46.345264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gen = generator(train_df, handcolumns, charmap, max_frame,max_char)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.347275Z","iopub.execute_input":"2024-01-10T10:27:46.347549Z","iopub.status.idle":"2024-01-10T10:27:46.356622Z","shell.execute_reply.started":"2024-01-10T10:27:46.347528Z","shell.execute_reply":"2024-01-10T10:27:46.355801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Shape and Data Type Check","metadata":{}},{"cell_type":"code","source":"# for (x,i1), i2 in gen:\n# break","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.358972Z","iopub.execute_input":"2024-01-10T10:27:46.359929Z","iopub.status.idle":"2024-01-10T10:27:46.36639Z","shell.execute_reply.started":"2024-01-10T10:27:46.359889Z","shell.execute_reply":"2024-01-10T10:27:46.365265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# i1.dtype, x.shape, i1.shape, i2.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.367921Z","iopub.execute_input":"2024-01-10T10:27:46.368294Z","iopub.status.idle":"2024-01-10T10:27:46.376224Z","shell.execute_reply.started":"2024-01-10T10:27:46.368261Z","shell.execute_reply":"2024-01-10T10:27:46.375367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del gen\n# del x\n# del i1\n# del i2","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.377265Z","iopub.execute_input":"2024-01-10T10:27:46.377506Z","iopub.status.idle":"2024-01-10T10:27:46.386668Z","shell.execute_reply.started":"2024-01-10T10:27:46.377484Z","shell.execute_reply":"2024-01-10T10:27:46.385749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Train & Test Dataset","metadata":{}},{"cell_type":"code","source":"signature = (\n (\n tf.TensorSpec(shape=(max_frame, len(handcolumns)), dtype=tf.float32),\n tf.TensorSpec(shape=(max_char, 1), dtype=tf.int8)\n ),\n tf.TensorSpec(shape=(max_char, 1), dtype=tf.int8)\n )","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.387798Z","iopub.execute_input":"2024-01-10T10:27:46.388231Z","iopub.status.idle":"2024-01-10T10:27:46.395935Z","shell.execute_reply.started":"2024-01-10T10:27:46.388208Z","shell.execute_reply":"2024-01-10T10:27:46.395005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size=32\ntrain_gen = generator(train_df, handcolumns, charmap, max_frame,max_char)\ntrain_dataset = tf.data.Dataset.from_generator(lambda : train_gen, output_types=((tf.float32, tf.int8), tf.int8))\ntrain_dataset = train_dataset.batch(batch_size)\n\ntest_gen = generator(test_df, handcolumns, charmap, max_frame,max_char)\ntest_dataset = tf.data.Dataset.from_generator(lambda : test_gen, output_types=((tf.float32, tf.int8), tf.int8))\ntest_dataset = test_dataset.batch(batch_size)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:46.397001Z","iopub.execute_input":"2024-01-10T10:27:46.397514Z","iopub.status.idle":"2024-01-10T10:27:47.578563Z","shell.execute_reply.started":"2024-01-10T10:27:46.397488Z","shell.execute_reply":"2024-01-10T10:27:47.577556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Shape & Datatype Check","metadata":{}},{"cell_type":"code","source":"# it = test_dataset.as_numpy_iterator()\n# x,y = it.next()\n# x[0].shape, x[1].shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:47.579744Z","iopub.execute_input":"2024-01-10T10:27:47.579998Z","iopub.status.idle":"2024-01-10T10:27:47.583954Z","shell.execute_reply.started":"2024-01-10T10:27:47.579976Z","shell.execute_reply":"2024-01-10T10:27:47.582941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del it\n# del x\n# del y","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:27:47.58538Z","iopub.execute_input":"2024-01-10T10:27:47.586303Z","iopub.status.idle":"2024-01-10T10:27:47.594952Z","shell.execute_reply.started":"2024-01-10T10:27:47.586264Z","shell.execute_reply":"2024-01-10T10:27:47.59411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Encoder-Decoder Architecture","metadata":{}},{"cell_type":"code","source":"units = 1024\n\n\n\n# Encoder\nenc_inp = Input(shape=(max_frame, len(handcolumns)))\n\nconv1d = Conv1D(units, 8, padding='same')\nconv1out = conv1d(enc_inp)\nmaxpool1d = MaxPool1D()\npoolout = maxpool1d(conv1out)\n\nconv1d2 = Conv1D(units, 5, padding='same')\nconv1out2 = conv1d2(poolout)\nmaxpool1d2 = MaxPool1D()\npoolout2 = maxpool1d2(conv1out2)\n\nbilstm = Bidirectional(LSTM(units, return_sequences=True), merge_mode='ave')\nbilstm_out = bilstm(poolout2)\ndropout_enc = Dropout(0.3)(bilstm_out)\nbilstm2 = Bidirectional(LSTM(units, return_sequences=True), merge_mode='ave')\nbilstm2_out = bilstm2(dropout_enc)\nbilstm3 = Bidirectional(LSTM(units, return_state=True), merge_mode='ave')\n\nskip_layer = tf.keras.layers.Add()\nadded = skip_layer([poolout2,bilstm2_out, dropout_enc])\n\nenc_x, f_h, f_c, b_h, b_c = bilstm3(added)\n\n# Transform States\ndense1 = Dense(units, activation='linear')\nb_hT = dense1(b_h)\ndense2 = Dense(units, activation='linear')\nb_cT = dense2(b_c)\n\n# Decoder\ndec_inp = Input(shape=(max_char,))\nemb_layer = Embedding(max(charmap.values())+1, units)\nembeddings = emb_layer(dec_inp)\n\nlstm_before = LSTM(units, return_sequences=True, return_state=True)\ndec_lstm_before_out, f_h1, f_c1 = lstm_before(embeddings,initial_state=[b_hT, b_cT])\ndropout_dec1 = Dropout(0.5)(dec_lstm_before_out)\nlstm_dec = LSTM(units, return_sequences=True, return_state=True)\ndec_lstm_out1, f_h2, f_c2 = lstm_dec(dropout_dec1, initial_state=[f_h1, f_c1])\ndropout_dec = Dropout(0.5)(dec_lstm_out1)\nlstm_dec2 = LSTM(units, return_sequences=True, return_state=True)\ndec_lstm_out2, dec_f_h, dec_f_c = lstm_dec2(dropout_dec, initial_state=[f_h2, f_c2])\n\n\ndec_skip_layer = tf.keras.layers.Add()\ndec_added = dec_skip_layer([dec_lstm_out1, dec_lstm_out2])\n\ndense = Dense(units, activation='relu')\ndense_out = dense(dec_added)\ndense2 = Dense(len(charmap), activation='linear')\ndense_out2 = dense2(dense_out)\n\nm = Model(inputs=[enc_inp, dec_inp], outputs=dense_out2)\nm.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:28:02.118327Z","iopub.execute_input":"2024-01-10T10:28:02.119048Z","iopub.status.idle":"2024-01-10T10:28:14.101774Z","shell.execute_reply.started":"2024-01-10T10:28:02.119014Z","shell.execute_reply":"2024-01-10T10:28:14.100782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Encoder Separator\n# sep_enc_inp = ","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:28:14.103436Z","iopub.execute_input":"2024-01-10T10:28:14.103732Z","iopub.status.idle":"2024-01-10T10:28:14.107705Z","shell.execute_reply.started":"2024-01-10T10:28:14.103706Z","shell.execute_reply":"2024-01-10T10:28:14.106693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(m, show_shapes=True, show_layer_activations=True, show_trainable=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:28:14.108906Z","iopub.execute_input":"2024-01-10T10:28:14.109227Z","iopub.status.idle":"2024-01-10T10:28:14.558077Z","shell.execute_reply.started":"2024-01-10T10:28:14.109203Z","shell.execute_reply":"2024-01-10T10:28:14.557156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Loss Function, Optimizer, Perfomance Metric","metadata":{}},{"cell_type":"code","source":"def masked_accuracy(y_true, y_pred):\n \"\"\"\n Calculate accuracy while masking out padded tokens in zero-padded sequences using TensorFlow.\n\n Parameters:\n - y_true: True labels, a 2D tensor of shape (batch_size, sequence_length) where padding is represented as 0.\n - y_pred: Predicted labels, a 2D tensor of the same shape as y_true.\n\n Returns:\n - accuracy: A scalar tensor representing the accuracy.\n \"\"\"\n\n # Create a mask for non-padded tokens by checking where y_true is not equal to 0.\n mask = tf.cast(tf.math.not_equal(y_true, 0), dtype=tf.float32)\n\n # Calculate the number of correctly classified non-padded tokens.\n correct_tokens = tf.reduce_sum(tf.cast(tf.math.equal(y_true, y_pred), dtype=tf.float32) * mask)\n\n # Calculate the total number of non-padded tokens in the batch.\n total_tokens = tf.reduce_sum(mask)\n\n # Calculate the accuracy by dividing the correct non-padded tokens by the total non-padded tokens.\n accuracy = correct_tokens / total_tokens\n\n return accuracy","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:28:25.218919Z","iopub.execute_input":"2024-01-10T10:28:25.219654Z","iopub.status.idle":"2024-01-10T10:28:25.226693Z","shell.execute_reply.started":"2024-01-10T10:28:25.219619Z","shell.execute_reply":"2024-01-10T10:28:25.225454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optmizer = tf.keras.optimizers.Adam(learning_rate=1e-3)\nlossfn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n# metrics = tf.keras.metrics.Accuracy()\nmetrics = masked_accuracy","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:28:27.128262Z","iopub.execute_input":"2024-01-10T10:28:27.128922Z","iopub.status.idle":"2024-01-10T10:28:27.137953Z","shell.execute_reply.started":"2024-01-10T10:28:27.128889Z","shell.execute_reply":"2024-01-10T10:28:27.137116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# m.compile(optimizer=optmizer, loss=lossfn, metrics=[metrics])","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:28:27.419182Z","iopub.execute_input":"2024-01-10T10:28:27.419912Z","iopub.status.idle":"2024-01-10T10:28:27.423951Z","shell.execute_reply.started":"2024-01-10T10:28:27.419879Z","shell.execute_reply":"2024-01-10T10:28:27.422908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train On Data","metadata":{}},{"cell_type":"code","source":"epochs = 20\ntrain_step_per_epoch = train_df.shape[0]//batch_size\nvalidation_step_per_epoch = test_df.shape[0]//batch_size","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:28:31.519159Z","iopub.execute_input":"2024-01-10T10:28:31.519793Z","iopub.status.idle":"2024-01-10T10:28:31.524696Z","shell.execute_reply.started":"2024-01-10T10:28:31.51976Z","shell.execute_reply":"2024-01-10T10:28:31.523603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ! pip install ipywidgets","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:28:32.218689Z","iopub.execute_input":"2024-01-10T10:28:32.219027Z","iopub.status.idle":"2024-01-10T10:28:32.22325Z","shell.execute_reply.started":"2024-01-10T10:28:32.219Z","shell.execute_reply":"2024-01-10T10:28:32.22215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_it = train_dataset.as_numpy_iterator()\nts_it = test_dataset.as_numpy_iterator()\n\nfor epoch in range(epochs):\n print(f\"Training for Epoch: {epoch+1}/{epochs}\")\n cum_loss = 0\n cum_acc = 0\n for i in (pbar:=tqdm(range(train_step_per_epoch))):\n x_batch, y_batch = tr_it.next()\n with tf.GradientTape() as tape:\n logits = m(x_batch, training=True)\n loss_value = lossfn(y_batch, logits)\n grads = tape.gradient(loss_value, m.trainable_weights)\n optmizer.apply_gradients(zip(grads, m.trainable_weights))\n acc = metrics(y_batch, tf.argmax(logits, axis=-1))\n\n cum_loss += loss_value.numpy()\n cum_acc += acc.numpy()\n \n lossstr = str(np.round(cum_loss/(i+1), 4))\n accstr = str(np.round(cum_acc/(i+1)*100,3))+\"%\"\n pbar.set_description(f\"Loss: {lossstr}, Accuracy: {accstr}\")\n print(f\"Avg. Loss{cum_loss/train_step_per_epoch}, Avg. Accuracy {cum_acc*100/train_step_per_epoch}%\")\n print(f\"Validating...\")\n cum_loss = 0\n cum_acc = 0\n for i in (pbar:=tqdm(range(validation_step_per_epoch))):\n x_batch, y_batch = ts_it.next()\n logits = m(x_batch, training=False) \n loss_value = lossfn(y_batch, logits)\n acc = metrics(y_batch, tf.argmax(logits, axis=-1))\n\n cum_loss += loss_value.numpy()\n cum_acc += acc.numpy()\n \n lossstr = str(np.round(cum_loss/(i+1), 4))\n accstr = str(np.round(cum_acc/(i+1)*100,3))+\"%\"\n pbar.set_description(f\"Loss: {lossstr}, Accuracy: {accstr}\")\n print(f\"Avg. Loss{cum_loss/validation_step_per_epoch}, Avg. Accuracy {cum_acc*100/validation_step_per_epoch}%\")\n print(\"==\"*70)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T10:28:32.734084Z","iopub.execute_input":"2024-01-10T10:28:32.73447Z","iopub.status.idle":"2024-01-10T15:42:15.519614Z","shell.execute_reply.started":"2024-01-10T10:28:32.734439Z","shell.execute_reply":"2024-01-10T15:42:15.51791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m.save(\"Full Model.h5\")","metadata":{"execution":{"iopub.status.busy":"2024-01-10T16:13:11.286371Z","iopub.execute_input":"2024-01-10T16:13:11.286752Z","iopub.status.idle":"2024-01-10T16:13:11.791895Z","shell.execute_reply.started":"2024-01-10T16:13:11.286721Z","shell.execute_reply":"2024-01-10T16:13:11.791044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_manager.keys()","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:42:22.006752Z","iopub.execute_input":"2024-01-10T15:42:22.007803Z","iopub.status.idle":"2024-01-10T15:42:22.015936Z","shell.execute_reply.started":"2024-01-10T15:42:22.007766Z","shell.execute_reply":"2024-01-10T15:42:22.014607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inverse_charmap = {v:k for k,v in charmap.items()}","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:42:22.755065Z","iopub.execute_input":"2024-01-10T15:42:22.756059Z","iopub.status.idle":"2024-01-10T15:42:22.760443Z","shell.execute_reply.started":"2024-01-10T15:42:22.756025Z","shell.execute_reply":"2024-01-10T15:42:22.759499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"texts = [\"\".join([inverse_charmap.get(i,\"['UNK']\") for i in seq if i not in [2,0]]) for seq in y_batch]\ntexts","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:42:24.052029Z","iopub.execute_input":"2024-01-10T15:42:24.052703Z","iopub.status.idle":"2024-01-10T15:42:24.073923Z","shell.execute_reply.started":"2024-01-10T15:42:24.05267Z","shell.execute_reply":"2024-01-10T15:42:24.073049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logits = m.predict(x_batch)\ntexts_pred = [\"\".join([inverse_charmap.get(i,\"['UNK']\") for i in seq if i not in [2,0]]) for seq in tf.argmax(logits, axis=-1).numpy()]","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:42:24.579605Z","iopub.execute_input":"2024-01-10T15:42:24.580274Z","iopub.status.idle":"2024-01-10T15:42:27.857903Z","shell.execute_reply.started":"2024-01-10T15:42:24.580243Z","shell.execute_reply":"2024-01-10T15:42:27.857128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"texts_pred","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:42:27.859787Z","iopub.execute_input":"2024-01-10T15:42:27.860544Z","iopub.status.idle":"2024-01-10T15:42:27.867052Z","shell.execute_reply.started":"2024-01-10T15:42:27.860507Z","shell.execute_reply":"2024-01-10T15:42:27.866192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for a,p in zip(texts, texts_pred):\n print(f\"Actual: {a} ===> Predicted: {p}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:42:45.172246Z","iopub.execute_input":"2024-01-10T15:42:45.172625Z","iopub.status.idle":"2024-01-10T15:42:45.178799Z","shell.execute_reply.started":"2024-01-10T15:42:45.172593Z","shell.execute_reply":"2024-01-10T15:42:45.177708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Develop Final Encoder-Decoder Separator Model","metadata":{}},{"cell_type":"markdown","source":"One dense layer is lost as same variable is used. Now we have to filter it by name","metadata":{}},{"cell_type":"code","source":"for layer in m.layers:\n if layer.name=='dense_1':\n break","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:58:17.984579Z","iopub.execute_input":"2024-01-10T15:58:17.984941Z","iopub.status.idle":"2024-01-10T15:58:17.989893Z","shell.execute_reply.started":"2024-01-10T15:58:17.984913Z","shell.execute_reply":"2024-01-10T15:58:17.988769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"layer.name","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:58:25.749059Z","iopub.execute_input":"2024-01-10T15:58:25.749724Z","iopub.status.idle":"2024-01-10T15:58:25.755469Z","shell.execute_reply.started":"2024-01-10T15:58:25.749692Z","shell.execute_reply":"2024-01-10T15:58:25.754527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Encoder\nenc_inp_model = Input(shape=(None, len(handcolumns)))\n\nconv1out_model = conv1d(enc_inp_model)\npoolout_model = maxpool1d(conv1out_model)\n\nconv1out2_model = conv1d2(poolout_model)\npoolout2_model = maxpool1d2(conv1out2_model)\n\nbilstm_out_model = bilstm(poolout2_model)\ndropout_enc_model = Dropout(0.3)(bilstm_out_model)\nbilstm2_out_model = bilstm2(bilstm_out_model)\n\nadded_model = skip_layer([poolout2_model,bilstm2_out_model, dropout_enc_model])\n\nenc_x_model, f_h_model, f_c_model, b_h_model, b_c_model = bilstm3(added_model)\n\n# Transform States\nb_hT_model = dense1(b_h_model)\nb_cT_model = layer(b_c_model)\n\nencoder_model = Model(enc_inp_model, outputs=[b_hT_model, b_cT_model])","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:58:35.088881Z","iopub.execute_input":"2024-01-10T15:58:35.089378Z","iopub.status.idle":"2024-01-10T15:58:36.505837Z","shell.execute_reply.started":"2024-01-10T15:58:35.089344Z","shell.execute_reply":"2024-01-10T15:58:36.505036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder_model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:58:36.507299Z","iopub.execute_input":"2024-01-10T15:58:36.507577Z","iopub.status.idle":"2024-01-10T15:58:36.547904Z","shell.execute_reply.started":"2024-01-10T15:58:36.507552Z","shell.execute_reply":"2024-01-10T15:58:36.547022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(encoder_model, show_shapes=True, show_layer_activations=True, show_trainable=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:58:40.007377Z","iopub.execute_input":"2024-01-10T15:58:40.007762Z","iopub.status.idle":"2024-01-10T15:58:40.131035Z","shell.execute_reply.started":"2024-01-10T15:58:40.007729Z","shell.execute_reply":"2024-01-10T15:58:40.130119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oh,oc = encoder_model.predict(x_batch[0])","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:58:44.170612Z","iopub.execute_input":"2024-01-10T15:58:44.170971Z","iopub.status.idle":"2024-01-10T15:58:46.17175Z","shell.execute_reply.started":"2024-01-10T15:58:44.170941Z","shell.execute_reply":"2024-01-10T15:58:46.170789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oh.shape, oc.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-10T15:58:46.173379Z","iopub.execute_input":"2024-01-10T15:58:46.173681Z","iopub.status.idle":"2024-01-10T15:58:46.179696Z","shell.execute_reply.started":"2024-01-10T15:58:46.173653Z","shell.execute_reply":"2024-01-10T15:58:46.178774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder_model.save(\"encoder.h5\")","metadata":{"execution":{"iopub.status.busy":"2024-01-10T16:13:36.720732Z","iopub.execute_input":"2024-01-10T16:13:36.721628Z","iopub.status.idle":"2024-01-10T16:13:37.047379Z","shell.execute_reply.started":"2024-01-10T16:13:36.721592Z","shell.execute_reply":"2024-01-10T16:13:37.04633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decoder\ndec_inp_model = Input(shape=(1,))\nb_hT_inp = Input(shape=(1024,))\nb_cT_inp = Input(shape=(1024,))\nembeddings_model = emb_layer(dec_inp_model)\ndec_lstm_before_out_model, f_h1_model, f_c1_model = lstm_before(embeddings_model,initial_state=[b_hT_inp, b_cT_inp])\ndropout_dec1_model = Dropout(0.5)(dec_lstm_before_out_model)\ndec_lstm_out1_model, f_h2_model, f_c2_model = lstm_dec(dropout_dec1_model, initial_state=[f_h1_model, f_c1_model])\ndropout_dec_model = Dropout(0.5)(dec_lstm_out1_model)\ndec_lstm_out2_model, dec_f_h_model, dec_f_c_model = lstm_dec2(dropout_dec_model, initial_state=[f_h2_model, f_c2_model])\n\n\ndec_added_model = dec_skip_layer([dec_lstm_out1_model, dec_lstm_out2_model])\n\ndense_out_model = dense(dec_added_model)\n# dense2 = Dense(len(charmap), activation='linear')\ndense_out2_model = dense2(dense_out_model)\ndecoder_model = Model(inputs=[dec_inp_model, b_hT_inp, b_cT_inp], outputs = [dense_out2_model, f_h1_model, f_c1_model])","metadata":{"execution":{"iopub.status.busy":"2024-01-10T16:07:41.958441Z","iopub.execute_input":"2024-01-10T16:07:41.959193Z","iopub.status.idle":"2024-01-10T16:07:42.686341Z","shell.execute_reply.started":"2024-01-10T16:07:41.959159Z","shell.execute_reply":"2024-01-10T16:07:42.685391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoder_model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-10T16:07:42.687937Z","iopub.execute_input":"2024-01-10T16:07:42.688241Z","iopub.status.idle":"2024-01-10T16:07:42.729497Z","shell.execute_reply.started":"2024-01-10T16:07:42.688214Z","shell.execute_reply":"2024-01-10T16:07:42.728673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(decoder_model, show_shapes=True, show_layer_activations=True, show_trainable=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-10T16:07:42.73069Z","iopub.execute_input":"2024-01-10T16:07:42.730948Z","iopub.status.idle":"2024-01-10T16:07:42.851127Z","shell.execute_reply.started":"2024-01-10T16:07:42.730925Z","shell.execute_reply":"2024-01-10T16:07:42.850259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoder_model.save(\"decoder.h5\")","metadata":{"execution":{"iopub.status.busy":"2024-01-10T16:13:51.631562Z","iopub.execute_input":"2024-01-10T16:13:51.632301Z","iopub.status.idle":"2024-01-10T16:13:51.809375Z","shell.execute_reply.started":"2024-01-10T16:13:51.632267Z","shell.execute_reply":"2024-01-10T16:13:51.808133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}