{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":52950,"databundleVersionId":5973250,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Dependences","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport numpy as np\nimport pandas as pd\nimport random\nimport spacy\nimport re\nimport pyarrow.parquet as pq\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport json\n\nimport torch\nfrom torch import nn\nfrom torch.utils.data import DataLoader,Dataset,random_split\nfrom torch.nn.utils.rnn import pad_sequence\nfrom torch.utils.tensorboard import SummaryWriter\n\nimport torchtext\nfrom torchtext.vocab import build_vocab_from_iterator\nfrom torchtext.datasets import Multi30k\nfrom torchtext.data.utils import get_tokenizer\nfrom torchtext.data.metrics import bleu_score\nfrom torchmetrics.text import BLEUScore\nfrom torchmetrics.text import WordErrorRate\nimport torchinfo\nfrom torchinfo import summary","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-10T11:26:17.715014Z","iopub.execute_input":"2024-02-10T11:26:17.715405Z","iopub.status.idle":"2024-02-10T11:26:17.724856Z","shell.execute_reply.started":"2024-02-10T11:26:17.715375Z","shell.execute_reply":"2024-02-10T11:26:17.723866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:19.48451Z","iopub.execute_input":"2024-02-10T11:26:19.484907Z","iopub.status.idle":"2024-02-10T11:26:19.491266Z","shell.execute_reply.started":"2024-02-10T11:26:19.484877Z","shell.execute_reply":"2024-02-10T11:26:19.490392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading Dataset","metadata":{}},{"cell_type":"code","source":"data_path = '/kaggle/input/asl-fingerspelling/supplemental_metadata.csv'\ndf = pd.read_csv(data_path)\ndf","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:20.424659Z","iopub.execute_input":"2024-02-10T11:26:20.425029Z","iopub.status.idle":"2024-02-10T11:26:20.507762Z","shell.execute_reply.started":"2024-02-10T11:26:20.424997Z","shell.execute_reply":"2024-02-10T11:26:20.50683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['phrase'][0]","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:21.084151Z","iopub.execute_input":"2024-02-10T11:26:21.084899Z","iopub.status.idle":"2024-02-10T11:26:21.090549Z","shell.execute_reply.started":"2024-02-10T11:26:21.084864Z","shell.execute_reply":"2024-02-10T11:26:21.089627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_sentence_len = max(len(x) for x in tqdm(df['phrase']) )\nmax_sentence_len","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:21.971673Z","iopub.execute_input":"2024-02-10T11:26:21.972043Z","iopub.status.idle":"2024-02-10T11:26:22.042312Z","shell.execute_reply.started":"2024-02-10T11:26:21.972012Z","shell.execute_reply":"2024-02-10T11:26:22.04136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_sentence_ids = df.sequence_id\nall_sentence_ids","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:22.255571Z","iopub.execute_input":"2024-02-10T11:26:22.255893Z","iopub.status.idle":"2024-02-10T11:26:22.263221Z","shell.execute_reply.started":"2024-02-10T11:26:22.255867Z","shell.execute_reply":"2024-02-10T11:26:22.262251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_id, file_id, phrase = df.iloc[1][['sequence_id', 'file_id', 'phrase']]\nsample_sequence_df = pq.read_table(f\"/kaggle/input/asl-fingerspelling/supplemental_landmarks/{str(33432165)}.parquet\",\n                                   filters=[[('sequence_id', '=', sequence_id)]]).to_pandas()\nprint(\"Full sequence dataset shape is {} -> (no.frames,all mediapipe landmarks)\".format(sample_sequence_df.shape))","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:22.86171Z","iopub.execute_input":"2024-02-10T11:26:22.862072Z","iopub.status.idle":"2024-02-10T11:26:24.186632Z","shell.execute_reply.started":"2024-02-10T11:26:22.862043Z","shell.execute_reply":"2024-02-10T11:26:24.18567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sequence_df","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:24.188376Z","iopub.execute_input":"2024-02-10T11:26:24.188902Z","iopub.status.idle":"2024-02-10T11:26:24.218447Z","shell.execute_reply.started":"2024-02-10T11:26:24.188866Z","shell.execute_reply":"2024-02-10T11:26:24.217479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sign Preprocessing","metadata":{}},{"cell_type":"markdown","source":"## Take Hands and Pose Landmarks","metadata":{}},{"cell_type":"code","source":"LPOSE = [13,15,17,19,21]\nRPOSE = [14,16,18,20,22]\nPOSE = LPOSE + RPOSE\nPOSE","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:25.898162Z","iopub.execute_input":"2024-02-10T11:26:25.898873Z","iopub.status.idle":"2024-02-10T11:26:25.909245Z","shell.execute_reply.started":"2024-02-10T11:26:25.898838Z","shell.execute_reply":"2024-02-10T11:26:25.908299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = [f'x_right_hand_{i}' for i in range(21)] + [f'x_left_hand_{i}' for i in range(21)] + [f'x_pose_{i}' for i in POSE]\nY = [f'y_right_hand_{i}' for i in range(21)] + [f'y_left_hand_{i}' for i in range(21)] + [f'y_pose_{i}' for i in POSE]\nZ = [f'z_right_hand_{i}' for i in range(21)] + [f'z_left_hand_{i}' for i in range(21)] + [f'z_pose_{i}' for i in POSE]\nFEATURE_COLUMNS = X+Y+Z","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:27.123162Z","iopub.execute_input":"2024-02-10T11:26:27.123859Z","iopub.status.idle":"2024-02-10T11:26:27.129889Z","shell.execute_reply.started":"2024-02-10T11:26:27.123803Z","shell.execute_reply":"2024-02-10T11:26:27.128859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(FEATURE_COLUMNS)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:27.814049Z","iopub.execute_input":"2024-02-10T11:26:27.814665Z","iopub.status.idle":"2024-02-10T11:26:27.820231Z","shell.execute_reply.started":"2024-02-10T11:26:27.814636Z","shell.execute_reply":"2024-02-10T11:26:27.819296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURE_COLUMNS[0],FEATURE_COLUMNS[21],FEATURE_COLUMNS[42]","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:28.476359Z","iopub.execute_input":"2024-02-10T11:26:28.476698Z","iopub.status.idle":"2024-02-10T11:26:28.482845Z","shell.execute_reply.started":"2024-02-10T11:26:28.476669Z","shell.execute_reply":"2024-02-10T11:26:28.481849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"x_\" in col]\nY_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"y_\" in col]\nZ_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"z_\" in col]\n\nRHAND_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if \"right\" in col]\nLHAND_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if  \"left\" in col]\nRPOSE_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if  \"pose\" in col and int(col[-2:]) in RPOSE]\nLPOSE_IDX = [i for i, col in enumerate(FEATURE_COLUMNS)  if  \"pose\" in col and int(col[-2:]) in LPOSE]","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:29.185238Z","iopub.execute_input":"2024-02-10T11:26:29.185576Z","iopub.status.idle":"2024-02-10T11:26:29.193022Z","shell.execute_reply.started":"2024-02-10T11:26:29.185547Z","shell.execute_reply":"2024-02-10T11:26:29.191905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(X_IDX),len(Y_IDX),len(Z_IDX)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:29.654199Z","iopub.execute_input":"2024-02-10T11:26:29.654887Z","iopub.status.idle":"2024-02-10T11:26:29.661291Z","shell.execute_reply.started":"2024-02-10T11:26:29.654855Z","shell.execute_reply":"2024-02-10T11:26:29.660353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(RHAND_IDX),len(LHAND_IDX),len(RPOSE_IDX),len(LPOSE_IDX)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:30.758536Z","iopub.execute_input":"2024-02-10T11:26:30.758937Z","iopub.status.idle":"2024-02-10T11:26:30.765157Z","shell.execute_reply.started":"2024-02-10T11:26:30.758904Z","shell.execute_reply":"2024-02-10T11:26:30.764193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_to_collect = []\n\nfor file_id in tqdm(df.file_id.unique()):\n    # Assuming the paths and filtering logic are already correctly defined\n    file_df = df.loc[df[\"file_id\"] == file_id]\n    parquet_df = pq.read_table(f\"/kaggle/input/asl-fingerspelling/supplemental_landmarks/{file_id}.parquet\",\n                               columns=['sequence_id'] + FEATURE_COLUMNS).to_pandas()\n    parquet_numpy = parquet_df.to_numpy()\n    # Loop through each sequence in file\n    for seq_id, phrase in zip(file_df.sequence_id, file_df.phrase):\n        # Assuming frames extraction logic is handled as per your dataset's requirements\n        \n        frames = parquet_numpy[parquet_df.index == seq_id]\n\n        # Calculate the number of NaN values in each hand landmark\n        r_nonan = np.sum(np.sum(np.isnan(frames[:, RHAND_IDX]), axis=1) == 0)\n        l_nonan = np.sum(np.sum(np.isnan(frames[:, LHAND_IDX]), axis=1) == 0)\n        no_nan = max(r_nonan, l_nonan)\n\n        if 2*len(phrase) < no_nan:\n            # Prepare data entry for each sequence\n            data_entry = {}\n            for i, column_name in enumerate(FEATURE_COLUMNS):\n                # Flatten and store each feature's values\n                data_entry[column_name] = frames[:, i].flatten()\n            # Store phrase as a feature\n            data_entry[\"text\"] = phrase\n\n            # Append this entry to our collection list\n            data_to_collect.append(data_entry)\n\n# Convert collected data into a DataFrame\nhands_df = pd.DataFrame(data_to_collect)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:26:32.516621Z","iopub.execute_input":"2024-02-10T11:26:32.517001Z","iopub.status.idle":"2024-02-10T11:27:28.544966Z","shell.execute_reply.started":"2024-02-10T11:26:32.516969Z","shell.execute_reply":"2024-02-10T11:27:28.544099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:27:28.546908Z","iopub.execute_input":"2024-02-10T11:27:28.547531Z","iopub.status.idle":"2024-02-10T11:27:28.852956Z","shell.execute_reply.started":"2024-02-10T11:27:28.547494Z","shell.execute_reply":"2024-02-10T11:27:28.852026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select columns for 'right' including specified pose columns\nright_hand_cols = [col for col in hands_df.columns if 'right_hand' in col]\nright_pose_cols = [f'{dim}_pose_{i}' for i in [14, 16, 18, 20, 22] for dim in ['x', 'y', 'z']]\nright_cols = right_hand_cols + right_pose_cols\n\n# Select columns for 'left' including specified pose columns\nleft_hand_cols = [col for col in hands_df.columns if 'left_hand' in col]\nleft_pose_cols = [f'{dim}_pose_{i}' for i in [13, 15, 17, 19, 21] for dim in ['x', 'y', 'z']]\nleft_cols = left_hand_cols + left_pose_cols\n\n# Function to concatenate values of given columns for a row\ndef concat_cols(row, cols):\n    return [row[col] for col in cols]\n\n# Concatenate values for 'right' and 'left'\nhands_df['right'] = hands_df.apply(concat_cols, cols=right_cols, axis=1)\nhands_df['left'] = hands_df.apply(concat_cols, cols=left_cols, axis=1)\n\n# Create a new DataFrame with desired columns\nhands_df = hands_df[[ 'right', 'left','text']].copy()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:27:28.854366Z","iopub.execute_input":"2024-02-10T11:27:28.854722Z","iopub.status.idle":"2024-02-10T11:27:54.230312Z","shell.execute_reply.started":"2024-02-10T11:27:28.854689Z","shell.execute_reply":"2024-02-10T11:27:54.229278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df['right'] = hands_df['right'].apply(lambda x: np.array(x).T)\nhands_df['left'] = hands_df['left'].apply(lambda x: np.array(x).T)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:27:54.232407Z","iopub.execute_input":"2024-02-10T11:27:54.23271Z","iopub.status.idle":"2024-02-10T11:28:00.396023Z","shell.execute_reply.started":"2024-02-10T11:27:54.232685Z","shell.execute_reply":"2024-02-10T11:28:00.395222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Resize and Pad Sequence","metadata":{}},{"cell_type":"code","source":"def resize_pad_array(arr, target_frames=128, num_features=78):\n    current_frames = arr.shape[0]\n    if current_frames > target_frames:\n        # Truncate the array if it has more than target_frames\n        return arr[:target_frames]\n    elif current_frames < target_frames:\n        # Pad the array with zeros if it has less than target_frames\n        padding = np.zeros((target_frames - current_frames, num_features))\n        return np.vstack([arr, padding])\n    else:\n        # Return the array as is if it already matches the target_frames\n        return arr\n\n# Apply the normalization function to each row for 'right' and 'left' columns\nfor index, row in tqdm(hands_df.iterrows()):\n    hands_df.at[index, 'right'] = resize_pad_array(row['right'])\n    hands_df.at[index, 'left'] = resize_pad_array(row['left'])\n\n# Verify the changes\nfor index, row in hands_df.iterrows():\n    print(f\"Right shape: {row['right'].shape}, Left shape: {row['left'].shape}\")\n    break","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:28:00.397132Z","iopub.execute_input":"2024-02-10T11:28:00.397427Z","iopub.status.idle":"2024-02-10T11:28:05.093114Z","shell.execute_reply.started":"2024-02-10T11:28:00.397402Z","shell.execute_reply":"2024-02-10T11:28:05.092171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:28:05.103492Z","iopub.execute_input":"2024-02-10T11:28:05.103852Z","iopub.status.idle":"2024-02-10T11:28:07.09451Z","shell.execute_reply.started":"2024-02-10T11:28:05.103801Z","shell.execute_reply":"2024-02-10T11:28:07.093507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Mirror Left Hand","metadata":{}},{"cell_type":"code","source":"frame_num = 128\nnum_rows = len(hands_df.text)\nfor row in tqdm(range(num_rows)):\n    \n    for frame in range(frame_num):\n        left_row_frame = hands_df.left[row][frame]\n        # [:21 x_hand,21:42 y_hand,42:63 z_hand,63:68 x_pose, 68:73 y_pose 73:78 z_pose]\n        left_row_frame[:21] = 1 -  left_row_frame[:21]\n        left_row_frame[63:68] = 1- left_row_frame[63:68] \n        hands_df.left[row][frame] = left_row_frame","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:28:07.09599Z","iopub.execute_input":"2024-02-10T11:28:07.096351Z","iopub.status.idle":"2024-02-10T11:31:31.42086Z","shell.execute_reply.started":"2024-02-10T11:28:07.096317Z","shell.execute_reply":"2024-02-10T11:31:31.419893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:31.422043Z","iopub.execute_input":"2024-02-10T11:31:31.422318Z","iopub.status.idle":"2024-02-10T11:31:33.301035Z","shell.execute_reply.started":"2024-02-10T11:31:31.422293Z","shell.execute_reply":"2024-02-10T11:31:33.300118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Take Dominant Hand","metadata":{}},{"cell_type":"code","source":"def dominant_hand(row):\n    lnan = np.count_nonzero(np.isnan(row['left']))\n    rnan = np.count_nonzero(np.isnan(row['right']))\n    if lnan < rnan:\n        return row['left']\n    else:\n        return row['right']","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:33.302326Z","iopub.execute_input":"2024-02-10T11:31:33.30264Z","iopub.status.idle":"2024-02-10T11:31:33.308156Z","shell.execute_reply.started":"2024-02-10T11:31:33.302584Z","shell.execute_reply":"2024-02-10T11:31:33.307132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df['sign'] = hands_df.apply(dominant_hand, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:33.311349Z","iopub.execute_input":"2024-02-10T11:31:33.311903Z","iopub.status.idle":"2024-02-10T11:31:34.89586Z","shell.execute_reply.started":"2024-02-10T11:31:33.31187Z","shell.execute_reply":"2024-02-10T11:31:34.894871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:34.896932Z","iopub.execute_input":"2024-02-10T11:31:34.897218Z","iopub.status.idle":"2024-02-10T11:31:37.829469Z","shell.execute_reply.started":"2024-02-10T11:31:34.897192Z","shell.execute_reply":"2024-02-10T11:31:37.828493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df = hands_df[['sign','text']].copy()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:37.830618Z","iopub.execute_input":"2024-02-10T11:31:37.830926Z","iopub.status.idle":"2024-02-10T11:31:37.843026Z","shell.execute_reply.started":"2024-02-10T11:31:37.8309Z","shell.execute_reply":"2024-02-10T11:31:37.842111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:37.844184Z","iopub.execute_input":"2024-02-10T11:31:37.844447Z","iopub.status.idle":"2024-02-10T11:31:38.857554Z","shell.execute_reply.started":"2024-02-10T11:31:37.844425Z","shell.execute_reply":"2024-02-10T11:31:38.856661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hands_df['sign'] = hands_df['sign'].apply(lambda x : x.reshape(-1))","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:38.858908Z","iopub.execute_input":"2024-02-10T11:31:38.859212Z","iopub.status.idle":"2024-02-10T11:31:38.86338Z","shell.execute_reply.started":"2024-02-10T11:31:38.859186Z","shell.execute_reply":"2024-02-10T11:31:38.862297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:38.864645Z","iopub.execute_input":"2024-02-10T11:31:38.865039Z","iopub.status.idle":"2024-02-10T11:31:39.83728Z","shell.execute_reply.started":"2024-02-10T11:31:38.865001Z","shell.execute_reply":"2024-02-10T11:31:39.836296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df.sign[0].shape","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:39.838436Z","iopub.execute_input":"2024-02-10T11:31:39.838797Z","iopub.status.idle":"2024-02-10T11:31:39.845243Z","shell.execute_reply.started":"2024-02-10T11:31:39.838764Z","shell.execute_reply":"2024-02-10T11:31:39.844202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Remove NaN","metadata":{}},{"cell_type":"code","source":"hands_df['sign'] = hands_df['sign'].apply(lambda x: np.where(np.isnan(x), 0, x))","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:39.846317Z","iopub.execute_input":"2024-02-10T11:31:39.846552Z","iopub.status.idle":"2024-02-10T11:31:41.756286Z","shell.execute_reply.started":"2024-02-10T11:31:39.84653Z","shell.execute_reply":"2024-02-10T11:31:41.755215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:31:41.757432Z","iopub.execute_input":"2024-02-10T11:31:41.75773Z","iopub.status.idle":"2024-02-10T11:31:42.718536Z","shell.execute_reply.started":"2024-02-10T11:31:41.757705Z","shell.execute_reply":"2024-02-10T11:31:42.717565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Text Preprocessing","metadata":{}},{"cell_type":"markdown","source":"## Char to Number","metadata":{}},{"cell_type":"code","source":"with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n    char_to_num = json.load(f)\n\n# Add pad_token, start pointer and end pointer to the dict\npad_token = 'P'\nstart_token = '<'\nend_token = '>'\npad_token_idx = 59\nstart_token_idx = 60\nend_token_idx = 61\n\nchar_to_num[pad_token] = pad_token_idx\nchar_to_num[start_token] = start_token_idx\nchar_to_num[end_token] = end_token_idx\nnum_to_char = {j:i for i,j in char_to_num.items()}","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:33:00.015346Z","iopub.execute_input":"2024-02-10T11:33:00.015694Z","iopub.status.idle":"2024-02-10T11:33:00.02736Z","shell.execute_reply.started":"2024-02-10T11:33:00.015667Z","shell.execute_reply":"2024-02-10T11:33:00.026541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(char_to_num)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:33:19.761997Z","iopub.execute_input":"2024-02-10T11:33:19.762777Z","iopub.status.idle":"2024-02-10T11:33:19.767548Z","shell.execute_reply.started":"2024-02-10T11:33:19.762744Z","shell.execute_reply":"2024-02-10T11:33:19.766512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(char_to_num)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:33:20.987923Z","iopub.execute_input":"2024-02-10T11:33:20.988662Z","iopub.status.idle":"2024-02-10T11:33:20.99421Z","shell.execute_reply.started":"2024-02-10T11:33:20.988623Z","shell.execute_reply":"2024-02-10T11:33:20.993351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df['text'] = hands_df['text'].apply(lambda x: start_token+x+end_token)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:33:25.359438Z","iopub.execute_input":"2024-02-10T11:33:25.360171Z","iopub.status.idle":"2024-02-10T11:33:25.383576Z","shell.execute_reply.started":"2024-02-10T11:33:25.360134Z","shell.execute_reply":"2024-02-10T11:33:25.382819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hands_df","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:33:30.957902Z","iopub.execute_input":"2024-02-10T11:33:30.958537Z","iopub.status.idle":"2024-02-10T11:33:31.919785Z","shell.execute_reply.started":"2024-02-10T11:33:30.958506Z","shell.execute_reply":"2024-02-10T11:33:31.918787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Dataset","metadata":{}},{"cell_type":"code","source":"class Sign2TextDataset(Dataset):\n    def __init__(self, dataframe):\n        \"\"\"\n        Args:\n            dataframe (DataFrame): DataFrame containing the signals and text.\n        \"\"\"\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n\n        sign = self.dataframe.iloc[idx, 0]\n        text = self.dataframe.iloc[idx, 1]\n\n        # If your signal data is not already a torch tensor, you might want to convert it\n        sign_tensor = torch.tensor(sign, dtype=torch.float32)\n\n        # Here, you would handle the text data appropriately.\n        # This example assumes you are returning the text as is, but you might\n        # want to apply some preprocessing like tokenization.\n        \n        sample = (sign_tensor, text)\n\n        return sample","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:12.971201Z","iopub.execute_input":"2024-02-10T11:34:12.97193Z","iopub.status.idle":"2024-02-10T11:34:12.979057Z","shell.execute_reply.started":"2024-02-10T11:34:12.971897Z","shell.execute_reply":"2024-02-10T11:34:12.978037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = Sign2TextDataset(dataframe=hands_df[:32000])","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:13.921995Z","iopub.execute_input":"2024-02-10T11:34:13.92268Z","iopub.status.idle":"2024-02-10T11:34:13.926994Z","shell.execute_reply.started":"2024-02-10T11:34:13.92265Z","shell.execute_reply":"2024-02-10T11:34:13.926036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.__len__()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:14.284807Z","iopub.execute_input":"2024-02-10T11:34:14.285401Z","iopub.status.idle":"2024-02-10T11:34:14.290759Z","shell.execute_reply.started":"2024-02-10T11:34:14.285373Z","shell.execute_reply":"2024-02-10T11:34:14.289877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign,txt = next(iter(dataset))\nprint(\"Sign : \",sign)\nprint(\"Text : \",txt)\nsign.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:14.599918Z","iopub.execute_input":"2024-02-10T11:34:14.60048Z","iopub.status.idle":"2024-02-10T11:34:14.678417Z","shell.execute_reply.started":"2024-02-10T11:34:14.600454Z","shell.execute_reply":"2024-02-10T11:34:14.677504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tokenization and Vectorization","metadata":{}},{"cell_type":"code","source":"def vectorization(data):\n    processed_data = []\n    for (sign , txt) in tqdm(data,total=len(list(data))) :\n        sign_tensor = torch.tensor(sign, dtype=torch.float32)\n        txt_tensor = torch.tensor([char_to_num[letter] for letter in txt],dtype=torch.long)\n        processed_data.append((sign_tensor,txt_tensor))\n    return processed_data","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:16.155406Z","iopub.execute_input":"2024-02-10T11:34:16.156118Z","iopub.status.idle":"2024-02-10T11:34:16.161734Z","shell.execute_reply.started":"2024-02-10T11:34:16.156084Z","shell.execute_reply":"2024-02-10T11:34:16.160804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorized_dataset = vectorization(dataset)\nlen(vectorized_dataset)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:16.819299Z","iopub.execute_input":"2024-02-10T11:34:16.81957Z","iopub.status.idle":"2024-02-10T11:34:26.307735Z","shell.execute_reply.started":"2024-02-10T11:34:16.819548Z","shell.execute_reply":"2024-02-10T11:34:26.30685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign,txt = next(iter(dataset))\nprint(\"Sign : \",sign)\nprint(\"Text : \",txt)\nvectorized_sign,vectorized_txt = next(iter(vectorized_dataset))\nprint(\"Vectorized Sign : \",vectorized_sign)\nprint(\"Vectorized Text : \",vectorized_txt)\nvectorized_letters = [num_to_char[int(x)] for x in vectorized_txt ]\nprint('Vectorized Letters :',vectorized_letters)\nvectorized_txt_back = ''\nfor letter in vectorized_letters :\n    vectorized_txt_back+=letter\nprint('Vectorized Text Back:',vectorized_txt_back)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:26.309551Z","iopub.execute_input":"2024-02-10T11:34:26.309877Z","iopub.status.idle":"2024-02-10T11:34:26.321132Z","shell.execute_reply.started":"2024-02-10T11:34:26.309847Z","shell.execute_reply":"2024-02-10T11:34:26.320229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_size = len(vectorized_dataset)\ntrain_size = int(total_size * 0.80)  # 80% of the entire dataset\ntest_size = total_size - train_size  # The remaining 20% for testing\n\n# Splitting the dataset\ntrain_dataset, test_dataset = random_split(vectorized_dataset, [train_size, test_size])","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:26.322327Z","iopub.execute_input":"2024-02-10T11:34:26.32263Z","iopub.status.idle":"2024-02-10T11:34:26.332411Z","shell.execute_reply.started":"2024-02-10T11:34:26.322567Z","shell.execute_reply":"2024-02-10T11:34:26.331475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_dataset),len(test_dataset)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:26.334091Z","iopub.execute_input":"2024-02-10T11:34:26.334345Z","iopub.status.idle":"2024-02-10T11:34:26.340347Z","shell.execute_reply.started":"2024-02-10T11:34:26.334323Z","shell.execute_reply":"2024-02-10T11:34:26.339378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Dataloader","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 64","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:26.341427Z","iopub.execute_input":"2024-02-10T11:34:26.341692Z","iopub.status.idle":"2024-02-10T11:34:26.348469Z","shell.execute_reply.started":"2024-02-10T11:34:26.34167Z","shell.execute_reply":"2024-02-10T11:34:26.347552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_fn(data):\n    sign_data, txt_data = [], []\n    \n    for sign_vector, txt_vector in data:\n        sign_data.append(torch.tensor(sign_vector))  # Ensure sign_vector is in the correct form\n        # Assuming txt_vector is already a tensor or a list that can be converted to a tensor\n        txt_data.append(torch.tensor(txt_vector, dtype=torch.long))  # Ensure correct type for txt_vector\n    \n    # Stack sign_data to create a batched tensor\n    sign_data = torch.stack(sign_data)\n    sign_data = sign_data.transpose(0, 1)\n    # Pad txt_data sequences\n    txt_data = pad_sequence(sequences=txt_data, padding_value=pad_token_idx)\n    \n    return sign_data, txt_data","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:27.070857Z","iopub.execute_input":"2024-02-10T11:34:27.071705Z","iopub.status.idle":"2024-02-10T11:34:27.079628Z","shell.execute_reply.started":"2024-02-10T11:34:27.071662Z","shell.execute_reply":"2024-02-10T11:34:27.07864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(dataset=train_dataset,batch_size=BATCH_SIZE,collate_fn=collate_fn,)\nvalid_dataloader = DataLoader(dataset=test_dataset,batch_size=BATCH_SIZE,collate_fn=collate_fn)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:32.970676Z","iopub.execute_input":"2024-02-10T11:34:32.971384Z","iopub.status.idle":"2024-02-10T11:34:32.976598Z","shell.execute_reply.started":"2024-02-10T11:34:32.971346Z","shell.execute_reply":"2024-02-10T11:34:32.97546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign,txt = next(iter(train_dataset))\nsign_batch,txt_batch = next(iter(train_dataloader))\n\nprint('txt : \\n',txt)\ntxt_vector=[]\nfor batch_word in txt_batch :\n    for word in batch_word:\n        txt_vector.append(word.item())\n        break\nprint('Text Vector : \\n',txt_vector)\ntext_letters = [num_to_char[int(x)] for x in txt_vector ]\nprint('Text Letters: \\n',text_letters)\ntxt_sentence = ''\nfor letter in text_letters :\n    txt_sentence+=letter\nprint('Text Sentence : \\n',txt_sentence)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:35.738043Z","iopub.execute_input":"2024-02-10T11:34:35.738887Z","iopub.status.idle":"2024-02-10T11:34:35.765077Z","shell.execute_reply.started":"2024-02-10T11:34:35.738846Z","shell.execute_reply":"2024-02-10T11:34:35.764172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign_batch.shape,txt_batch.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:36.276938Z","iopub.execute_input":"2024-02-10T11:34:36.277275Z","iopub.status.idle":"2024-02-10T11:34:36.283084Z","shell.execute_reply.started":"2024-02-10T11:34:36.27725Z","shell.execute_reply":"2024-02-10T11:34:36.282194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn.Embedding(9984*64,300)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:34:37.11548Z","iopub.execute_input":"2024-02-10T11:34:37.115886Z","iopub.status.idle":"2024-02-10T11:34:39.081545Z","shell.execute_reply.started":"2024-02-10T11:34:37.115841Z","shell.execute_reply":"2024-02-10T11:34:39.080597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Our Model","metadata":{}},{"cell_type":"markdown","source":"## LandMarks Embedding Layer","metadata":{}},{"cell_type":"code","source":"class LandmarkEmbedding(nn.Module):\n    def __init__(self, num_hid=64, maxlen=100):\n        super(LandmarkEmbedding, self).__init__()\n        # Define the convolutional layers\n        self.conv1 = nn.Conv1d(in_channels=num_hid, out_channels=num_hid, kernel_size=11, stride=2, padding=5)\n        self.conv2 = nn.Conv1d(in_channels=num_hid, out_channels=num_hid, kernel_size=11, stride=2, padding=5)\n        self.conv3 = nn.Conv1d(in_channels=num_hid, out_channels=num_hid, kernel_size=11, stride=2, padding=5)\n        \n        # Define the positional embedding layer\n        self.pos_emb = nn.Embedding(num_embeddings=maxlen, embedding_dim=num_hid)\n        \n        # Activation function\n        self.relu = nn.ReLU()\n\n    def forward(self, x):\n        # Apply the convolutional layers with ReLU activation\n#         x = x.permute(0 , 2 , 1)\n        x = self.relu(self.conv1(x))\n        x = self.relu(self.conv2(x))\n        x = self.relu(self.conv3(x))\n#         x = x.permute(0 , 2 , 1)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:58:22.642585Z","iopub.execute_input":"2024-02-10T11:58:22.643015Z","iopub.status.idle":"2024-02-10T11:58:22.651548Z","shell.execute_reply.started":"2024-02-10T11:58:22.642982Z","shell.execute_reply":"2024-02-10T11:58:22.650612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_embedding = LandmarkEmbedding()\ntest_embedding","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:58:27.087929Z","iopub.execute_input":"2024-02-10T11:58:27.088311Z","iopub.status.idle":"2024-02-10T11:58:27.097349Z","shell.execute_reply.started":"2024-02-10T11:58:27.088282Z","shell.execute_reply":"2024-02-10T11:58:27.096427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Encoder","metadata":{}},{"cell_type":"code","source":"sign_batch.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:58:28.870625Z","iopub.execute_input":"2024-02-10T11:58:28.871014Z","iopub.status.idle":"2024-02-10T11:58:28.877456Z","shell.execute_reply.started":"2024-02-10T11:58:28.870981Z","shell.execute_reply":"2024-02-10T11:58:28.876537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign_batch.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:58:33.870037Z","iopub.execute_input":"2024-02-10T11:58:33.870418Z","iopub.status.idle":"2024-02-10T11:58:33.876585Z","shell.execute_reply.started":"2024-02-10T11:58:33.870389Z","shell.execute_reply":"2024-02-10T11:58:33.875657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_embedding(sign_batch).shape","metadata":{"execution":{"iopub.status.busy":"2024-02-10T11:58:35.886067Z","iopub.execute_input":"2024-02-10T11:58:35.886712Z","iopub.status.idle":"2024-02-10T11:58:35.90026Z","shell.execute_reply.started":"2024-02-10T11:58:35.886679Z","shell.execute_reply":"2024-02-10T11:58:35.899352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Encoder(nn.Module):\n    def __init__(self,input_size,hidden_size,num_layers,dropout_ratio):\n        super(Encoder,self).__init__()\n        # intialization of hyperparameters\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.num_layers = num_layers\n        self.dropout_ratio = dropout_ratio\n        \n        # intialization of layers\n        self.dropout_layer = nn.Dropout(dropout_ratio)\n        self.embedding_layer = LandmarkEmbedding()\n        self.rnn = nn.LSTM(input_size=10,hidden_size=hidden_size,num_layers=num_layers,dropout=dropout_ratio)\n    \n    def forward(self,x:torch.Tensor)->torch.Tensor :\n#         x = x.transpose(0, 1).unsqueeze(1)\n        embedding = self.dropout_layer(self.embedding_layer(x))\n#         embedding = embedding.squeeze(1).transpose(0, 2)\n        out,(hidden,cell) = self.rnn(embedding)\n        \n        \n        return hidden,cell\n        \n        ","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:21.985309Z","iopub.execute_input":"2024-02-10T12:02:21.986261Z","iopub.status.idle":"2024-02-10T12:02:21.993613Z","shell.execute_reply.started":"2024-02-10T12:02:21.986222Z","shell.execute_reply":"2024-02-10T12:02:21.992653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_encoder = Encoder(input_size=64,hidden_size=1024,num_layers=2,dropout_ratio=0.5)\ntest_encoder","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:22.571648Z","iopub.execute_input":"2024-02-10T12:02:22.572322Z","iopub.status.idle":"2024-02-10T12:02:22.696633Z","shell.execute_reply.started":"2024-02-10T12:02:22.572284Z","shell.execute_reply":"2024-02-10T12:02:22.695734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(model=test_encoder,input_data=sign_batch)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:23.739034Z","iopub.execute_input":"2024-02-10T12:02:23.739396Z","iopub.status.idle":"2024-02-10T12:02:24.72876Z","shell.execute_reply.started":"2024-02-10T12:02:23.739366Z","shell.execute_reply":"2024-02-10T12:02:24.727793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Decoder","metadata":{}},{"cell_type":"code","source":"class Decoder(nn.Module):\n    def __init__(self,input_size,embedding_size,hidden_size,num_layers,output_size,dropout_ratio):\n        super(Decoder,self).__init__()\n        self.input_size = input_size\n        self.embedding_size = embedding_size\n        self.hidden_size = hidden_size\n        self.num_layers = num_layers\n        self.output_size = output_size\n        self.dropout_ratio = dropout_ratio\n        \n        self.dropout_layer = nn.Dropout(dropout_ratio)\n        self.embedding_layer = nn.Embedding(num_embeddings=input_size,embedding_dim=embedding_size)\n        self.rnn = nn.LSTM(input_size=embedding_size,hidden_size=hidden_size,num_layers=num_layers,dropout=dropout_ratio)\n        self.fc = nn.Linear(in_features=hidden_size,out_features=output_size)\n        \n    def forward(self,x:torch.Tensor,hidden,cell)->torch.Tensor:\n        x = x.unsqueeze(0)\n        embedding = self.dropout_layer(self.embedding_layer(x))\n        out,(hidden,cell) = self.rnn(embedding,(hidden,cell))\n        predictions = self.fc(out)\n        predictions = predictions.squeeze(0)\n        \n        return predictions,hidden,cell","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:26.007408Z","iopub.execute_input":"2024-02-10T12:02:26.008287Z","iopub.status.idle":"2024-02-10T12:02:26.016948Z","shell.execute_reply.started":"2024-02-10T12:02:26.008255Z","shell.execute_reply":"2024-02-10T12:02:26.015846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_decoder = Decoder(input_size=len(char_to_num),embedding_size = len(char_to_num),hidden_size=1024,num_layers=2,output_size=len(char_to_num),dropout_ratio=0.5)\ntest_decoder","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:27.026232Z","iopub.execute_input":"2024-02-10T12:02:27.026992Z","iopub.status.idle":"2024-02-10T12:02:27.149283Z","shell.execute_reply.started":"2024-02-10T12:02:27.026958Z","shell.execute_reply":"2024-02-10T12:02:27.148321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_hidden,test_cell = test_encoder(sign_batch) ","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:28.019791Z","iopub.execute_input":"2024-02-10T12:02:28.020535Z","iopub.status.idle":"2024-02-10T12:02:29.255424Z","shell.execute_reply.started":"2024-02-10T12:02:28.0205Z","shell.execute_reply":"2024-02-10T12:02:29.254598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(model=test_decoder,input_data=(txt_batch[0],test_hidden,test_cell))","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:29.256877Z","iopub.execute_input":"2024-02-10T12:02:29.257167Z","iopub.status.idle":"2024-02-10T12:02:29.295282Z","shell.execute_reply.started":"2024-02-10T12:02:29.257142Z","shell.execute_reply":"2024-02-10T12:02:29.294229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Seq2Seq ","metadata":{}},{"cell_type":"code","source":"class Seq2Seq(nn.Module):\n    def __init__(self,enocder,decoder):\n        super(Seq2Seq,self).__init__()\n        self.encoder = enocder\n        self.decoder = decoder\n        \n    def forward(self,source,target,teacher_force_ratio=0.5):\n        batch_size = source.shape[1]\n        target_len = target.shape[0]\n        target_vocab_size = len(char_to_num)\n        \n        outputs = torch.zeros(target_len,batch_size,target_vocab_size)\n        \n        hidden,cell = self.encoder(source)\n        \n        x = target[0]\n        for t in range(1,target_len):\n            output,hidden,cell = self.decoder(x,hidden,cell)\n            \n            outputs[t] = output\n            best_gess = output.argmax(1)\n            \n            x = target[t] if random.random() < teacher_force_ratio else best_gess\n            \n        return outputs","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:30.731591Z","iopub.execute_input":"2024-02-10T12:02:30.732433Z","iopub.status.idle":"2024-02-10T12:02:30.740349Z","shell.execute_reply.started":"2024-02-10T12:02:30.7324Z","shell.execute_reply":"2024-02-10T12:02:30.739391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_seq2seq = Seq2Seq(test_encoder,test_decoder)\ntest_seq2seq","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:30.935404Z","iopub.execute_input":"2024-02-10T12:02:30.935707Z","iopub.status.idle":"2024-02-10T12:02:30.943322Z","shell.execute_reply.started":"2024-02-10T12:02:30.935683Z","shell.execute_reply":"2024-02-10T12:02:30.942393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"txt_batch[0].shape","metadata":{"execution":{"iopub.status.busy":"2024-02-10T13:43:58.531294Z","iopub.execute_input":"2024-02-10T13:43:58.531643Z","iopub.status.idle":"2024-02-10T13:43:58.538135Z","shell.execute_reply.started":"2024-02-10T13:43:58.531617Z","shell.execute_reply":"2024-02-10T13:43:58.53707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(model=test_seq2seq,input_data=(sign_batch,txt_batch,0.5))","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:31.140798Z","iopub.execute_input":"2024-02-10T12:02:31.141127Z","iopub.status.idle":"2024-02-10T12:02:32.738529Z","shell.execute_reply.started":"2024-02-10T12:02:31.141099Z","shell.execute_reply":"2024-02-10T12:02:32.737737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Our Model","metadata":{}},{"cell_type":"markdown","source":"## Encoder HyperParameters","metadata":{}},{"cell_type":"code","source":"num_hid=64\nencoder_input_size = num_hid\nencoder_hidden_size = 1024\nencoder_num_layers = 2\nencoder_dropout_ratio = 0.5","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:32.739953Z","iopub.execute_input":"2024-02-10T12:02:32.740236Z","iopub.status.idle":"2024-02-10T12:02:32.744785Z","shell.execute_reply.started":"2024-02-10T12:02:32.740212Z","shell.execute_reply":"2024-02-10T12:02:32.743867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = Encoder(input_size=encoder_input_size,\n                  hidden_size=encoder_hidden_size,\n                  num_layers=encoder_num_layers,\n                  dropout_ratio=encoder_dropout_ratio).to(device)\nencoder","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:32.746038Z","iopub.execute_input":"2024-02-10T12:02:32.746962Z","iopub.status.idle":"2024-02-10T12:02:32.878573Z","shell.execute_reply.started":"2024-02-10T12:02:32.746929Z","shell.execute_reply":"2024-02-10T12:02:32.877797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Decoder HyperParameters","metadata":{}},{"cell_type":"code","source":"decoder_input_size = len(char_to_num)\ndecoder_embedding_size = len(char_to_num)\ndecoder_hidden_size = 1024\ndecoder_num_layers = 2\ndecoder_output_size = len(char_to_num)\ndecoder_dropout_ratio = 0.5","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:32.880273Z","iopub.execute_input":"2024-02-10T12:02:32.880564Z","iopub.status.idle":"2024-02-10T12:02:32.885394Z","shell.execute_reply.started":"2024-02-10T12:02:32.880538Z","shell.execute_reply":"2024-02-10T12:02:32.884492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoder = Decoder(input_size=decoder_input_size,\n                  embedding_size = decoder_embedding_size,\n                  hidden_size=decoder_hidden_size,\n                  num_layers=decoder_num_layers,\n                  output_size=decoder_output_size,\n                  dropout_ratio=decoder_dropout_ratio).to(device)\ndecoder","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:32.886586Z","iopub.execute_input":"2024-02-10T12:02:32.886948Z","iopub.status.idle":"2024-02-10T12:02:33.007792Z","shell.execute_reply.started":"2024-02-10T12:02:32.886915Z","shell.execute_reply":"2024-02-10T12:02:33.006948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Seq2Seq(enocder=encoder,decoder=decoder).to(device)\nmodel","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:33.009026Z","iopub.execute_input":"2024-02-10T12:02:33.00937Z","iopub.status.idle":"2024-02-10T12:02:33.018311Z","shell.execute_reply.started":"2024-02-10T12:02:33.009338Z","shell.execute_reply":"2024-02-10T12:02:33.0173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model HyperParameters","metadata":{}},{"cell_type":"code","source":"learning_rate = 0.001\nepochs=20","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:33.058339Z","iopub.execute_input":"2024-02-10T12:02:33.058924Z","iopub.status.idle":"2024-02-10T12:02:33.06272Z","shell.execute_reply.started":"2024-02-10T12:02:33.058895Z","shell.execute_reply":"2024-02-10T12:02:33.061755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loss Function and Optimizer","metadata":{}},{"cell_type":"code","source":"loss_fn = nn.CrossEntropyLoss(ignore_index=pad_token_idx)\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:34.319735Z","iopub.execute_input":"2024-02-10T12:02:34.320089Z","iopub.status.idle":"2024-02-10T12:02:34.325451Z","shell.execute_reply.started":"2024-02-10T12:02:34.320063Z","shell.execute_reply":"2024-02-10T12:02:34.3245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_model = False\nsave_model = True","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:36.112297Z","iopub.execute_input":"2024-02-10T12:02:36.112648Z","iopub.status.idle":"2024-02-10T12:02:36.117025Z","shell.execute_reply.started":"2024-02-10T12:02:36.112614Z","shell.execute_reply":"2024-02-10T12:02:36.115964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tensorboard to get nice loss plot\nwriter = SummaryWriter(f\"runs/seq2seq attention/loss_plot\")\nstep = 0","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:02:37.189024Z","iopub.execute_input":"2024-02-10T12:02:37.189722Z","iopub.status.idle":"2024-02-10T12:02:37.195477Z","shell.execute_reply.started":"2024-02-10T12:02:37.189688Z","shell.execute_reply":"2024-02-10T12:02:37.194575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper Functions","metadata":{}},{"cell_type":"code","source":"def save_checkpoint(state, filename=\"my_checkpoint.pth.tar\"):\n    print(\"=> Saving checkpoint\")\n    torch.save(state, filename)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:03:14.43019Z","iopub.execute_input":"2024-02-10T12:03:14.430916Z","iopub.status.idle":"2024-02-10T12:03:14.435437Z","shell.execute_reply.started":"2024-02-10T12:03:14.430877Z","shell.execute_reply":"2024-02-10T12:03:14.434468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_checkpoint(checkpoint, model, optimizer):\n    print(\"=> Loading checkpoint\")\n    model.load_state_dict(checkpoint[\"state_dict\"])\n    optimizer.load_state_dict(checkpoint[\"optimizer\"])","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:03:15.108126Z","iopub.execute_input":"2024-02-10T12:03:15.10895Z","iopub.status.idle":"2024-02-10T12:03:15.113647Z","shell.execute_reply.started":"2024-02-10T12:03:15.108917Z","shell.execute_reply":"2024-02-10T12:03:15.112569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Loop","metadata":{}},{"cell_type":"code","source":"def train_step (model,train_dataloader,optimizer,loss_fn,device,step):\n    model.to(device)\n    train_loss = 0\n    \n    model.train()\n    model.to(device)\n    \n    for i,(source,target) in enumerate(tqdm(train_dataloader,desc='Trainig Loop : ')):\n        \n        source,target = source.to(device),target.to(device)\n        \n        # training loss\n        output = model(source,target)\n        output = output[1:].reshape(-1,output.shape[2])\n        target = target[1:].reshape(-1)\n        loss = loss_fn(output.to(device),target.to(device))\n        train_loss += loss\n        \n        # get optimizer ready\n        optimizer.zero_grad()\n        \n        # back propagation\n        loss.backward()\n        \n        # weights clipping to avoid exploding\n        torch.nn.utils.clip_grad_norm(parameters=model.parameters(),max_norm=1)\n        \n        # gradient descent\n        optimizer.step()\n        \n        # plot to tensorboard\n        writer.add_scalar(\"Training loss\", loss, global_step=step)\n        step += 1\n        \n        \n    train_loss = train_loss/len(train_dataloader)\n    \n    return train_loss    ","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:03:15.946999Z","iopub.execute_input":"2024-02-10T12:03:15.948091Z","iopub.status.idle":"2024-02-10T12:03:15.956752Z","shell.execute_reply.started":"2024-02-10T12:03:15.948056Z","shell.execute_reply":"2024-02-10T12:03:15.955709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing Loop","metadata":{}},{"cell_type":"code","source":"def test_step(model,test_dataloader,loss_fn,device):\n    model.to(device)\n    test_loss = 0\n\n    model.eval()\n    all_target_vectors = []\n    all_output_vectors = []\n    for i,(source,target) in enumerate(tqdm(valid_dataloader,desc='Testing Loop : ')):\n        source,target = source.to(device),target.to(device)\n        with torch.inference_mode():\n            output = model(source,target)\n            \n            target_vector=[]\n            for batch_word in target :\n                for word in batch_word:\n                    target_vector.append(word.item())\n                    break\n            output_vector=[]\n            for batch_word in output.argmax(2) :\n                for word in batch_word:\n                    output_vector.append(word.item())\n                    break\n            all_target_vectors.append(target_vector)\n            all_output_vectors.append(output_vector)\n            \n            output = output.reshape(-1,output.shape[2])\n            target = target.reshape(-1)\n            loss = loss_fn(output.to(device),target.to(device))\n            test_loss += loss\n    all_target_sentences = []\n    for target_vector in all_target_vectors:\n        target_letters = [num_to_char[int(x)] for x in target_vector ]\n        target_sentence = ''\n        for letter in target_letters :\n            target_sentence+=letter\n        all_target_sentences.append(target_sentence)\n\n    all_output_sentences = []\n    for output_vector in all_output_vectors:    \n        output_letters = [num_to_char[int(x)] for x in output_vector ]\n        output_sentence = '<'\n        for i,letter in enumerate(output_letters) :\n            if i == 0:\n                continue\n                \n            output_sentence+=letter\n            \n            if letter =='>':\n                break\n        all_output_sentences.append(output_sentence)\n        \n    for i in range(len(all_target_sentences[:10])):    \n        print(f'Original Sentence  {i+1} : ',all_target_sentences[i])\n        print(f'Predicted Sentence {i+1} : ',all_output_sentences[i])\n    \n    test_loss = test_loss/len(valid_dataloader)\n\n    return test_loss\n    ","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:03:16.334459Z","iopub.execute_input":"2024-02-10T12:03:16.335061Z","iopub.status.idle":"2024-02-10T12:03:16.347723Z","shell.execute_reply.started":"2024-02-10T12:03:16.335027Z","shell.execute_reply":"2024-02-10T12:03:16.346766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ALL in One (Train and Test Loop)","metadata":{}},{"cell_type":"code","source":"def train(model,train_dataloader,test_dataloader,epochs,optimizer,loss_fn,device):\n    load_model = False\n    if load_model:\n        load_checkpoint(torch.load(\"my_checkpoint.pth.tar\"), model, optimizer)\n\n    results ={\n        'train_loss':[],\n        'test_loss':[],\n    }\n    model.to(device)\n    epoch = 0\n    step = 0\n    best_metric_value = float('inf')  # Initialize with a large value for loss or a small value for accuracy\n    best_model_state_dict = None\n    for epoch in tqdm(range(epochs) ,desc=f'Epoch :'):\n        train_loss = train_step(model=model,\n                                train_dataloader=train_dataloader,\n                                optimizer=optimizer,\n                                loss_fn=loss_fn,\n                                device=device,step=step)\n        \n        test_loss = test_step(model=model,\n                                test_dataloader=test_dataloader,\n                                loss_fn=loss_fn,\n                                device=device)\n        \n        if test_loss < best_metric_value:\n            best_metric_value = test_loss\n            best_model_state_dict = model.state_dict()\n            best_model_path = 'best_Seq2Seq_model.pth'\n            torch.save(best_model_state_dict, best_model_path)\n            \n        results['train_loss'].append(train_loss)\n        results['test_loss'].append(test_loss)\n        print(f'Epoch : {epoch+1} | Train loss : {train_loss:.4f} | Test loss : {test_loss:.4f}')\n\n        \n        checkpoint = {\"state_dict\": model.state_dict(), \"optimizer\": optimizer.state_dict()}\n        save_checkpoint(checkpoint)\n\n        model.eval()\n        if save_model:\n            checkpoint = {\n                \"state_dict\": model.state_dict(),\n                \"optimizer\": optimizer.state_dict(),\n            }\n            save_checkpoint(checkpoint)\n    \n    return results\n    ","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:03:16.824356Z","iopub.execute_input":"2024-02-10T12:03:16.824884Z","iopub.status.idle":"2024-02-10T12:03:16.836987Z","shell.execute_reply.started":"2024-02-10T12:03:16.824837Z","shell.execute_reply":"2024-02-10T12:03:16.835237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = train(model=model,\n                train_dataloader=train_dataloader,\n                test_dataloader=valid_dataloader,\n                epochs=epochs,\n                optimizer=optimizer,\n                loss_fn=loss_fn,\n                device=device)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:03:16.942434Z","iopub.execute_input":"2024-02-10T12:03:16.943292Z","iopub.status.idle":"2024-02-10T12:34:20.107095Z","shell.execute_reply.started":"2024-02-10T12:03:16.94326Z","shell.execute_reply":"2024-02-10T12:34:20.105863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Loss Curves","metadata":{}},{"cell_type":"code","source":"def plot_loss_curves(results: dict[str, list[float]]):\n    \"\"\"Plots training curves of a results dictionary.\n\n    Args:\n        results (dict): dictionary containing list of values, e.g.\n            {\"train_loss\": [...],\n             \"train_accuracy\": [...],\n             \"test_loss\": [...],\n             \"test_accuracy\": [...]}\n    \"\"\"\n    \n    # Get the loss values of the results dictionary (training and test)\n    loss = results['train_loss']\n    test_loss = results['test_loss']\n\n    \n\n    # Figure out how many epochs there were\n    epochs = range(len(results['train_loss']))\n\n    # Setup a plot \n    plt.figure(figsize=(15, 7))\n    loss = [x.cpu().detach().numpy() for x in loss]\n    test_loss = [x.cpu().detach().numpy() for x in test_loss]\n    # Plot loss\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs, loss, label='train_loss')\n    plt.plot(epochs, test_loss, label='test_loss')\n    plt.title('Loss')\n    plt.xlabel('Epochs')\n    plt.legend()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:34:40.71811Z","iopub.execute_input":"2024-02-10T12:34:40.71848Z","iopub.status.idle":"2024-02-10T12:34:40.726408Z","shell.execute_reply.started":"2024-02-10T12:34:40.718451Z","shell.execute_reply":"2024-02-10T12:34:40.725286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_loss_curves(results)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:34:41.995976Z","iopub.execute_input":"2024-02-10T12:34:41.996695Z","iopub.status.idle":"2024-02-10T12:34:42.330212Z","shell.execute_reply.started":"2024-02-10T12:34:41.996665Z","shell.execute_reply":"2024-02-10T12:34:42.329152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = Seq2Seq(encoder,decoder)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:34:55.158465Z","iopub.execute_input":"2024-02-10T12:34:55.158902Z","iopub.status.idle":"2024-02-10T12:34:55.163781Z","shell.execute_reply.started":"2024-02-10T12:34:55.158862Z","shell.execute_reply":"2024-02-10T12:34:55.162674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_point = torch.load('best_Seq2Seq_model.pth')","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:34:56.090282Z","iopub.execute_input":"2024-02-10T12:34:56.091202Z","iopub.status.idle":"2024-02-10T12:34:56.623914Z","shell.execute_reply.started":"2024-02-10T12:34:56.091165Z","shell.execute_reply":"2024-02-10T12:34:56.62288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_point.keys()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:34:56.669113Z","iopub.execute_input":"2024-02-10T12:34:56.669416Z","iopub.status.idle":"2024-02-10T12:34:56.675724Z","shell.execute_reply.started":"2024-02-10T12:34:56.669389Z","shell.execute_reply":"2024-02-10T12:34:56.674669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model.load_state_dict(check_point)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:34:57.268335Z","iopub.execute_input":"2024-02-10T12:34:57.2687Z","iopub.status.idle":"2024-02-10T12:34:57.276656Z","shell.execute_reply.started":"2024-02-10T12:34:57.26867Z","shell.execute_reply":"2024-02-10T12:34:57.275621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model","metadata":{"execution":{"iopub.status.busy":"2024-02-10T12:34:59.735381Z","iopub.execute_input":"2024-02-10T12:34:59.736218Z","iopub.status.idle":"2024-02-10T12:34:59.742155Z","shell.execute_reply.started":"2024-02-10T12:34:59.736184Z","shell.execute_reply":"2024-02-10T12:34:59.741184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bleu Score","metadata":{}},{"cell_type":"code","source":"\ndef generate(data_loader):\n    best_model.to(device)\n\n    best_model.eval()\n    all_target_sentences = []\n    all_output_sentences = []\n    for i,(source,target) in enumerate(tqdm(data_loader)):\n        source,target = source.to(device),target.to(device)\n        with torch.inference_mode():\n            output = best_model(source,target)\n            \n            target_sentences = []\n            output_sentences = []\n#             if i == 0 :\n#                 print('output before',output.shape)\n            output = output.permute(1,0,2)\n#             if i == 0 :\n#                 print('output permute',output.shape)\n            output = output.argmax(2)\n#             if i == 0 :\n#                 print('output argmax',output.shape)\n            target = target.transpose(1,0)\n#             if i == 0 :\n#                 print('target after',output.shape)\n            for idx in range(target.shape[0]) : \n                target_text = ''.join([num_to_char[_] for _ in target[idx , :].cpu().detach().numpy() if _ != pad_token_idx]) \n                output_text = ''.join([num_to_char[_] for _ in output[idx , :].cpu().detach().numpy() if _ != end_token_idx]) \n                \n                target_sentences.append(target_text) \n                output_sentences.append(output_text) \n            \n            all_target_sentences.extend(target_sentences) \n            all_output_sentences.extend(output_sentences) \n            \n    all_target_sentences = np.array(all_target_sentences)\n    all_output_sentences = np.array(all_output_sentences) \n    \n    return all_target_sentences , all_output_sentences\n            ","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:02:00.224719Z","iopub.execute_input":"2024-02-10T16:02:00.225684Z","iopub.status.idle":"2024-02-10T16:02:00.235454Z","shell.execute_reply.started":"2024-02-10T16:02:00.225647Z","shell.execute_reply":"2024-02-10T16:02:00.234575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_trg_sentences , val_preds_sentences = generate(valid_dataloader)\ntrain_trg_sentences , train_preds_sentences = generate(train_dataloader)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:45:15.023662Z","iopub.execute_input":"2024-02-10T16:45:15.02404Z","iopub.status.idle":"2024-02-10T16:45:56.703855Z","shell.execute_reply.started":"2024-02-10T16:45:15.024008Z","shell.execute_reply":"2024-02-10T16:45:56.702871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds_sentences[0][1:len(val_trg_sentences[0])-1]","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:56:48.284763Z","iopub.execute_input":"2024-02-10T16:56:48.285545Z","iopub.status.idle":"2024-02-10T16:56:48.292511Z","shell.execute_reply.started":"2024-02-10T16:56:48.285513Z","shell.execute_reply":"2024-02-10T16:56:48.291211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_trg_sentences[0][1:-1]","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:58:08.029971Z","iopub.execute_input":"2024-02-10T16:58:08.030356Z","iopub.status.idle":"2024-02-10T16:58:08.036683Z","shell.execute_reply.started":"2024-02-10T16:58:08.030325Z","shell.execute_reply":"2024-02-10T16:58:08.035645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_trg_sentences.shape , val_preds_sentences.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:45:56.705495Z","iopub.execute_input":"2024-02-10T16:45:56.705787Z","iopub.status.idle":"2024-02-10T16:45:56.711671Z","shell.execute_reply.started":"2024-02-10T16:45:56.705759Z","shell.execute_reply":"2024-02-10T16:45:56.710767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_trg_sentences.shape , train_preds_sentences.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-10T17:03:07.033428Z","iopub.execute_input":"2024-02-10T17:03:07.034336Z","iopub.status.idle":"2024-02-10T17:03:07.040266Z","shell.execute_reply.started":"2024-02-10T17:03:07.034301Z","shell.execute_reply":"2024-02-10T17:03:07.039353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions","metadata":{}},{"cell_type":"code","source":"pad_val_trg_sentences = []\npad_val_preds_sentences = []\nfor i in tqdm(range(len(val_trg_sentences))):\n    pad_val_trg_sentences.append(val_trg_sentences[i][1:-1])\n    pad_val_preds_sentences.append(val_preds_sentences[i][1:len(val_trg_sentences[i])-1])","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:02:55.408513Z","iopub.execute_input":"2024-02-10T16:02:55.408932Z","iopub.status.idle":"2024-02-10T16:02:55.452893Z","shell.execute_reply.started":"2024-02-10T16:02:55.408898Z","shell.execute_reply":"2024-02-10T16:02:55.451257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pad_train_trg_sentences = []\npad_train_preds_sentences = []\nfor i in tqdm(range(len(train_trg_sentences))):\n    pad_train_trg_sentences.append(train_trg_sentences[i][1:-1])\n    pad_train_preds_sentences.append(train_preds_sentences[i][1:len(train_trg_sentences[i])-1])","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:39:35.808128Z","iopub.execute_input":"2024-02-10T16:39:35.808602Z","iopub.status.idle":"2024-02-10T16:39:35.912439Z","shell.execute_reply.started":"2024-02-10T16:39:35.808559Z","shell.execute_reply":"2024-02-10T16:39:35.911556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds_sentences[0][1:len(train_trg_sentences[0])-1]","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:58:52.018715Z","iopub.execute_input":"2024-02-10T16:58:52.01911Z","iopub.status.idle":"2024-02-10T16:58:52.025711Z","shell.execute_reply.started":"2024-02-10T16:58:52.019079Z","shell.execute_reply":"2024-02-10T16:58:52.024742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pad_train_trg_sentences[]","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:59:58.414073Z","iopub.execute_input":"2024-02-10T16:59:58.414737Z","iopub.status.idle":"2024-02-10T16:59:58.4205Z","shell.execute_reply.started":"2024-02-10T16:59:58.414704Z","shell.execute_reply":"2024-02-10T16:59:58.419572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_trg_sentences[:-][:]","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:44:57.215084Z","iopub.execute_input":"2024-02-10T16:44:57.21545Z","iopub.status.idle":"2024-02-10T16:44:57.221853Z","shell.execute_reply.started":"2024-02-10T16:44:57.21542Z","shell.execute_reply":"2024-02-10T16:44:57.220972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds_sentences","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:43:47.69011Z","iopub.execute_input":"2024-02-10T16:43:47.69049Z","iopub.status.idle":"2024-02-10T16:43:47.697202Z","shell.execute_reply.started":"2024-02-10T16:43:47.690458Z","shell.execute_reply":"2024-02-10T16:43:47.696074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds_sentences[:][:len(train_trg_sentences[2:-2])-1]","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:42:51.3225Z","iopub.execute_input":"2024-02-10T16:42:51.322909Z","iopub.status.idle":"2024-02-10T16:42:51.329676Z","shell.execute_reply.started":"2024-02-10T16:42:51.322876Z","shell.execute_reply":"2024-02-10T16:42:51.328726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_trg_sentences[:][2:-2]","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:39:47.44946Z","iopub.execute_input":"2024-02-10T16:39:47.450321Z","iopub.status.idle":"2024-02-10T16:39:47.457643Z","shell.execute_reply.started":"2024-02-10T16:39:47.450284Z","shell.execute_reply":"2024-02-10T16:39:47.456676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5):\n    print(f'Orginal Sentence {i+1}   : {pad_val_trg_sentences[i]}')\n    print(f'Predicted Sentence {i+1} : {pad_val_preds_sentences[i]}')\n    print()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:03:03.184759Z","iopub.execute_input":"2024-02-10T16:03:03.185541Z","iopub.status.idle":"2024-02-10T16:03:03.190762Z","shell.execute_reply.started":"2024-02-10T16:03:03.185507Z","shell.execute_reply":"2024-02-10T16:03:03.18986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5):\n    print(f'Orginal Sentence {i+1}   : {pad_train_trg_sentences[i]}')\n    print(f'Predicted Sentence {i+1} : {pad_train_preds_sentences[i]}')\n    print()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:32:24.945964Z","iopub.execute_input":"2024-02-10T16:32:24.946865Z","iopub.status.idle":"2024-02-10T16:32:24.953702Z","shell.execute_reply.started":"2024-02-10T16:32:24.946806Z","shell.execute_reply":"2024-02-10T16:32:24.952721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchmetrics.text import BLEUScore\nbleu_val = BLEUScore()\nbleu_train = BLEUScore()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:29:55.019774Z","iopub.execute_input":"2024-02-10T16:29:55.020182Z","iopub.status.idle":"2024-02-10T16:29:55.029579Z","shell.execute_reply.started":"2024-02-10T16:29:55.020148Z","shell.execute_reply":"2024-02-10T16:29:55.028367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bleu_val.update(val_preds_sentences , [val_trg_sentences])  ","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:52:56.327198Z","iopub.execute_input":"2024-02-10T16:52:56.327566Z","iopub.status.idle":"2024-02-10T16:52:59.029073Z","shell.execute_reply.started":"2024-02-10T16:52:56.327537Z","shell.execute_reply":"2024-02-10T16:52:59.028137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig_, ax_ = bleu_val.plot() ","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:53:02.164474Z","iopub.execute_input":"2024-02-10T16:53:02.164867Z","iopub.status.idle":"2024-02-10T16:53:02.28762Z","shell.execute_reply.started":"2024-02-10T16:53:02.164827Z","shell.execute_reply":"2024-02-10T16:53:02.286741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bleu_train.update(np.array(train_preds_sentences) , np.array([train_trg_sentences]))  ","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:53:20.510617Z","iopub.execute_input":"2024-02-10T16:53:20.511511Z","iopub.status.idle":"2024-02-10T16:53:31.447546Z","shell.execute_reply.started":"2024-02-10T16:53:20.511475Z","shell.execute_reply":"2024-02-10T16:53:31.446474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig_, ax_ = bleu_train.plot() ","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:53:31.44937Z","iopub.execute_input":"2024-02-10T16:53:31.44974Z","iopub.status.idle":"2024-02-10T16:53:31.573434Z","shell.execute_reply.started":"2024-02-10T16:53:31.449705Z","shell.execute_reply":"2024-02-10T16:53:31.572475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bleu_train_score = bleu_train(pad_train_preds_sentences , [pad_train_trg_sentences])","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:30:22.578874Z","iopub.execute_input":"2024-02-10T16:30:22.579888Z","iopub.status.idle":"2024-02-10T16:30:43.420043Z","shell.execute_reply.started":"2024-02-10T16:30:22.579854Z","shell.execute_reply":"2024-02-10T16:30:43.419221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bleu_val_score= bleu_val(pad_val_preds_sentences , [pad_val_trg_sentences])","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:30:43.421648Z","iopub.execute_input":"2024-02-10T16:30:43.421965Z","iopub.status.idle":"2024-02-10T16:30:48.373152Z","shell.execute_reply.started":"2024-02-10T16:30:43.421939Z","shell.execute_reply":"2024-02-10T16:30:48.372297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'BLEU Train Score : {bleu_train_score.item()*100:.2f} % ')\nprint(f'BLEU VAL Score   : {bleu_val_score.item()*100:.2f} % ')","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:30:49.824606Z","iopub.execute_input":"2024-02-10T16:30:49.825276Z","iopub.status.idle":"2024-02-10T16:30:49.830494Z","shell.execute_reply.started":"2024-02-10T16:30:49.825243Z","shell.execute_reply":"2024-02-10T16:30:49.829515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchmetrics.text import WordErrorRate\nWER_val = WordErrorRate()\nWER_train = WordErrorRate()\n#This value indicates the percentage of words that were incorrectly predicted.\n#The lower the value,\n#the better the performance of the ASR system with a WER of 0 being a perfect score","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:08:04.010638Z","iopub.execute_input":"2024-02-10T16:08:04.011498Z","iopub.status.idle":"2024-02-10T16:08:04.017809Z","shell.execute_reply.started":"2024-02-10T16:08:04.01146Z","shell.execute_reply":"2024-02-10T16:08:04.016883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WER_val.update(val_preds_sentences , val_trg_sentences)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:08:21.167828Z","iopub.execute_input":"2024-02-10T16:08:21.168675Z","iopub.status.idle":"2024-02-10T16:08:21.481845Z","shell.execute_reply.started":"2024-02-10T16:08:21.168638Z","shell.execute_reply":"2024-02-10T16:08:21.481014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig_, ax_ = WER_val.plot()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:08:34.353351Z","iopub.execute_input":"2024-02-10T16:08:34.353708Z","iopub.status.idle":"2024-02-10T16:08:34.481359Z","shell.execute_reply.started":"2024-02-10T16:08:34.35368Z","shell.execute_reply":"2024-02-10T16:08:34.480508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WER_train.update(train_preds_sentences , train_trg_sentences)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:09:16.526195Z","iopub.execute_input":"2024-02-10T16:09:16.526969Z","iopub.status.idle":"2024-02-10T16:09:17.770825Z","shell.execute_reply.started":"2024-02-10T16:09:16.526935Z","shell.execute_reply":"2024-02-10T16:09:17.769839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig_, ax_ = WER_train.plot()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:09:25.308541Z","iopub.execute_input":"2024-02-10T16:09:25.309009Z","iopub.status.idle":"2024-02-10T16:09:25.454679Z","shell.execute_reply.started":"2024-02-10T16:09:25.308967Z","shell.execute_reply":"2024-02-10T16:09:25.453516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WER_val_score = WER_val(val_preds_sentences , val_trg_sentences)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:15:12.46994Z","iopub.execute_input":"2024-02-10T16:15:12.470693Z","iopub.status.idle":"2024-02-10T16:15:12.776318Z","shell.execute_reply.started":"2024-02-10T16:15:12.470659Z","shell.execute_reply":"2024-02-10T16:15:12.775562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WER_train_score = WER_val(train_preds_sentences , train_trg_sentences)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:12:12.056961Z","iopub.execute_input":"2024-02-10T16:12:12.057378Z","iopub.status.idle":"2024-02-10T16:12:13.273823Z","shell.execute_reply.started":"2024-02-10T16:12:12.057344Z","shell.execute_reply":"2024-02-10T16:12:13.272785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'WER Train Score : {WER_train_score.item()*100:.2f} % ')\nprint(f'WER VAL Score   : {WER_val_score.item()*100:.2f} % ')","metadata":{"execution":{"iopub.status.busy":"2024-02-10T16:15:16.109741Z","iopub.execute_input":"2024-02-10T16:15:16.110678Z","iopub.status.idle":"2024-02-10T16:15:16.115507Z","shell.execute_reply.started":"2024-02-10T16:15:16.110627Z","shell.execute_reply":"2024-02-10T16:15:16.114529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}