{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport sklearn\nimport matplotlib.pyplot as plt\nimport json\nimport regex\nimport os\nfrom tqdm.notebook import tqdm\nimport math\nimport sys\nimport random\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import KFold\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.utils.data import *\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-10T10:00:01.219675Z","iopub.execute_input":"2023-07-10T10:00:01.220025Z","iopub.status.idle":"2023-07-10T10:00:04.833549Z","shell.execute_reply.started":"2023-07-10T10:00:01.219997Z","shell.execute_reply":"2023-07-10T10:00:04.832584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read Character to Ordinal Encoding Mapping\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    char2ord = json.load(json_file)\n\ndisplay(pd.Series(char2ord).to_frame('Ordinal Encoding'))","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:04.835306Z","iopub.execute_input":"2023-07-10T10:00:04.835952Z","iopub.status.idle":"2023-07-10T10:00:04.860438Z","shell.execute_reply.started":"2023-07-10T10:00:04.835899Z","shell.execute_reply":"2023-07-10T10:00:04.859594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames=np.load('/kaggle/input/aslr-hands/X.npy', allow_pickle=True)\nphrases = np.load('/kaggle/input/aslr-hands/Y.npy', allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:04.861716Z","iopub.execute_input":"2023-07-10T10:00:04.862335Z","iopub.status.idle":"2023-07-10T10:00:43.412486Z","shell.execute_reply.started":"2023-07-10T10:00:04.8623Z","shell.execute_reply":"2023-07-10T10:00:43.411503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GLOBAL CONFIG","metadata":{}},{"cell_type":"code","source":"# Ensure deterministic behavior\ntorch.backends.cudnn.deterministic = True\nrandom.seed(hash(\"setting random seeds\") % 2**32 - 1)\nnp.random.seed(hash(\"improves reproducibility\") % 2**32 - 1)\ntorch.manual_seed(hash(\"by removing stochasticity\") % 2**32 - 1)\ntorch.cuda.manual_seed_all(hash(\"so runs are repeatable\") % 2**32 - 1)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:43.41728Z","iopub.execute_input":"2023-07-10T10:00:43.417585Z","iopub.status.idle":"2023-07-10T10:00:43.428567Z","shell.execute_reply.started":"2023-07-10T10:00:43.417556Z","shell.execute_reply":"2023-07-10T10:00:43.427579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = dict(\n    NUM_EPOCHS = 50,\n    BATCH_SIZE =32,\n    d_model = 512,\n    ffn_hidden = 2048,\n    num_heads = 8,\n    drop_prob = 0.1,\n    num_layers = 2,\n    learning_rate = 1e-3,\n    dataset=\"ASLR\",\n    arcitecture = \"transformer\"\n             )","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:43.43004Z","iopub.execute_input":"2023-07-10T10:00:43.430404Z","iopub.status.idle":"2023-07-10T10:00:43.437676Z","shell.execute_reply.started":"2023-07-10T10:00:43.430373Z","shell.execute_reply":"2023-07-10T10:00:43.436369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 500\nNUM_EPOCHS = 50\nSTART_TOKEN = '<START>'\nEND_TOKEN = '<END>'\nPADDING_TOKEN = '<PAD>'\nBATCH_SIZE =32\nd_model = 512\nffn_hidden = 2048\nnum_heads = 8\ndrop_prob = 0.1\nnum_layers = 2\nlearning_rate = 1e-4\nk=5\nload_model = False\nnum_columns = frames[0].shape[1]\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:43.439591Z","iopub.execute_input":"2023-07-10T10:00:43.439875Z","iopub.status.idle":"2023-07-10T10:00:43.470882Z","shell.execute_reply.started":"2023-07-10T10:00:43.439853Z","shell.execute_reply":"2023-07-10T10:00:43.470012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sos_eos_pad= {START_TOKEN: len(char2ord), END_TOKEN: len(char2ord)+1, PADDING_TOKEN: len(char2ord)+2}\nchar2ord.update(sos_eos_pad)\n\nphrase_vocab_size = len(char2ord)\n\ndisplay(char2ord)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:43.472767Z","iopub.execute_input":"2023-07-10T10:00:43.473031Z","iopub.status.idle":"2023-07-10T10:00:43.491908Z","shell.execute_reply.started":"2023-07-10T10:00:43.473008Z","shell.execute_reply":"2023-07-10T10:00:43.491028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_word_indices = [char2ord[token] for token in list(phrases[0])]\n\nsentence_word_indices.insert(0, char2ord[START_TOKEN])\nsentence_word_indices.append(char2ord[END_TOKEN])\nfor _ in range(len(sentence_word_indices), MAX_SEQUENCE_LENGTH):\n    sentence_word_indices.append(char2ord[PADDING_TOKEN])","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:43.493233Z","iopub.execute_input":"2023-07-10T10:00:43.493603Z","iopub.status.idle":"2023-07-10T10:00:43.501629Z","shell.execute_reply.started":"2023-07-10T10:00:43.493573Z","shell.execute_reply":"2023-07-10T10:00:43.50079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sequence_pad(frame):\n    if len(frame)<MAX_SEQUENCE_LENGTH:\n        # Specify the number of empty rows to add\n        num_empty_rows = MAX_SEQUENCE_LENGTH-len(frame)\n        # Create empty rows filled with zeros\n        empty_rows = torch.zeros((num_empty_rows, frame.shape[1])).to(device)\n\n        # Concatenate the empty rows to the original array\n        frame = torch.cat((frame, empty_rows), dim=0)\n    else:\n        frame = frame[:MAX_SEQUENCE_LENGTH]\n        \n    # Check for NaN values in the tensor\n    nan_mask = torch.isnan(frame)\n\n    # Replace NaN values with 0 using torch.where\n    frame = torch.where(nan_mask, torch.tensor(0.0), frame)\n    return(frame)\n\ndef sequence_normalize(frame):\n    # Check for NaN values in the tensor\n    nan_mask = torch.isnan(frame)\n\n    # Replace NaN values with 0 using torch.where\n    frame = torch.where(nan_mask, torch.tensor(0.0), frame)\n\n    # Calculate the mean and standard deviation along the desired dimension (column-wise)\n    mean = torch.mean(frame, dim=0)\n    std = torch.std(frame, dim=0)\n\n    # Normalize the data by subtracting the mean and dividing by the standard deviation\n    normalized_frame = (frame-mean)/std\n    return normalized_frame","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:43.503013Z","iopub.execute_input":"2023-07-10T10:00:43.503681Z","iopub.status.idle":"2023-07-10T10:00:43.514707Z","shell.execute_reply.started":"2023-07-10T10:00:43.503651Z","shell.execute_reply":"2023-07-10T10:00:43.5138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NEG_INFTY = -1e9\n\ndef create_masks(frame, phrase):\n    look_ahead_mask = torch.full([MAX_SEQUENCE_LENGTH, MAX_SEQUENCE_LENGTH] , True) # Creates a tensor with all values = True\n    #print(look_ahead_mask)\n    look_ahead_mask = torch.triu(look_ahead_mask, diagonal=1) # Upper traingle = True only\n    #print(look_ahead_mask)\n    encoder_padding_mask = torch.full([MAX_SEQUENCE_LENGTH, MAX_SEQUENCE_LENGTH] , False)\n    decoder_padding_mask_self_attention = torch.full([MAX_SEQUENCE_LENGTH, MAX_SEQUENCE_LENGTH] , False)\n    decoder_padding_mask_cross_attention = torch.full([MAX_SEQUENCE_LENGTH, MAX_SEQUENCE_LENGTH] , False)\n    #print(encoder_padding_mask)\n\n    frame_length, eng_sentence_length = len(frame), len(phrase)\n    frame_chars_to_padding_mask = np.arange(frame_length + 1, MAX_SEQUENCE_LENGTH)\n    eng_chars_to_padding_mask = np.arange(eng_sentence_length + 1, MAX_SEQUENCE_LENGTH)\n    encoder_padding_mask[:, frame_chars_to_padding_mask] = True\n    encoder_padding_mask[frame_chars_to_padding_mask, :] = True\n    decoder_padding_mask_self_attention[:, eng_chars_to_padding_mask] = True\n    decoder_padding_mask_self_attention[eng_chars_to_padding_mask, :] = True\n    decoder_padding_mask_cross_attention[:, eng_chars_to_padding_mask] = True\n    decoder_padding_mask_cross_attention[eng_chars_to_padding_mask, :] = True\n\n    encoder_self_attention_mask = torch.where(encoder_padding_mask, NEG_INFTY, 0)\n    decoder_self_attention_mask =  torch.where(look_ahead_mask + decoder_padding_mask_self_attention, NEG_INFTY, 0)\n    decoder_cross_attention_mask = torch.where(decoder_padding_mask_cross_attention, NEG_INFTY, 0)\n    return encoder_self_attention_mask, decoder_self_attention_mask, decoder_cross_attention_mask","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:43.519503Z","iopub.execute_input":"2023-07-10T10:00:43.519799Z","iopub.status.idle":"2023-07-10T10:00:43.53124Z","shell.execute_reply.started":"2023-07-10T10:00:43.519776Z","shell.execute_reply":"2023-07-10T10:00:43.530286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset():\n    def __init__(self, frames, targets):\n        self.frames = frames\n        self.targets = targets\n    \n    def __len__(self):\n        return self.frames.shape[0]\n    \n    def tokenize_and_pad(self, target, use_start):\n        sentence_word_indices = np.array([char2ord[token] for token in list(target.lower())])\n        if use_start:\n            sentence_word_indices= np.insert(sentence_word_indices, 0, char2ord[START_TOKEN])\n        sentence_word_indices = np.append(sentence_word_indices, char2ord[END_TOKEN])\n        for _ in range(len(sentence_word_indices), MAX_SEQUENCE_LENGTH):\n            sentence_word_indices = np.append(sentence_word_indices, char2ord[PADDING_TOKEN])\n\n        return sentence_word_indices\n\n    def __getitem__(self, idx):\n        frame = torch.tensor(np.array(self.frames[idx])).to(device)\n        phrase = self.targets[idx]\n        encoder_self_attention_mask, decoder_self_attention_mask, decoder_cross_attention_mask = create_masks(frame, phrase)\n        frame = sequence_normalize(frame)\n        frame = sequence_pad(frame)\n        input_phrase = self.tokenize_and_pad(phrase, use_start = True)\n        target = self.tokenize_and_pad(phrase, use_start = False)\n        \n        return {\n            'frame': frame,\n            'input_phrase': torch.tensor(input_phrase).to(device),\n            'target': torch.tensor(target).to(device),\n            'encoder_self_attention_mask' : encoder_self_attention_mask.to(device), \n            'decoder_self_attention_mask' : decoder_self_attention_mask.to(device),\n            'decoder_cross_attention_mask' : decoder_cross_attention_mask.to(device) \n            \n        }","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:43.532988Z","iopub.execute_input":"2023-07-10T10:00:43.533246Z","iopub.status.idle":"2023-07-10T10:00:43.546586Z","shell.execute_reply.started":"2023-07-10T10:00:43.533224Z","shell.execute_reply":"2023-07-10T10:00:43.545589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = CustomDataset(frames, phrases)\nprint('Processed frame shape:',dataset[0]['frame'].shape)\nprint('Processed input_phrase shape:',dataset[0]['input_phrase'].shape)\nprint('Processed target shape:',dataset[0]['target'].shape)\nprint('encoder_self_attention_mask shape:',dataset[0]['encoder_self_attention_mask'].shape)\nprint('decoder_self_attention_mask shape:',dataset[0]['decoder_self_attention_mask'].shape)\nprint('decoder_cross_attention_mask shape:',dataset[0]['decoder_cross_attention_mask'].shape)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:43.548052Z","iopub.execute_input":"2023-07-10T10:00:43.549179Z","iopub.status.idle":"2023-07-10T10:00:46.835834Z","shell.execute_reply.started":"2023-07-10T10:00:43.549149Z","shell.execute_reply":"2023-07-10T10:00:46.834826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataloader = DataLoader(dataset=dataset, batch_size = BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:46.837222Z","iopub.execute_input":"2023-07-10T10:00:46.837668Z","iopub.status.idle":"2023-07-10T10:00:46.843573Z","shell.execute_reply.started":"2023-07-10T10:00:46.837632Z","shell.execute_reply":"2023-07-10T10:00:46.842569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch in dataloader:\n    print('Frames shape of a batch:',batch['frame'].shape)\n    print('Input phrase shape of a batch:',batch['input_phrase'].shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:46.845368Z","iopub.execute_input":"2023-07-10T10:00:46.84593Z","iopub.status.idle":"2023-07-10T10:00:47.407862Z","shell.execute_reply.started":"2023-07-10T10:00:46.845899Z","shell.execute_reply":"2023-07-10T10:00:47.406885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PositionalEncoding(nn.Module):\n    def __init__(self, d_model, max_sequence_length):\n        super().__init__()\n        self.max_sequence_length = max_sequence_length\n        self.d_model = d_model\n\n    def forward(self):\n        even_i = torch.arange(0, self.d_model, 2).float()\n        denominator = torch.pow(10000, even_i/self.d_model)\n        position = (torch.arange(self.max_sequence_length)\n                          .reshape(self.max_sequence_length, 1))\n        even_PE = torch.sin(position / denominator)\n        odd_PE = torch.cos(position / denominator)\n        stacked = torch.stack([even_PE, odd_PE], dim=2)\n        PE = torch.flatten(stacked, start_dim=1, end_dim=2)\n        return PE","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:47.40947Z","iopub.execute_input":"2023-07-10T10:00:47.409851Z","iopub.status.idle":"2023-07-10T10:00:47.417828Z","shell.execute_reply.started":"2023-07-10T10:00:47.409819Z","shell.execute_reply":"2023-07-10T10:00:47.416845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomEmbedding(nn.Module):\n    def __init__(self, max_sequence_length, num_columns, d_model, encoder_embed):\n        super().__init__()\n        self.max_sequence_length = max_sequence_length\n        self.num_columns = num_columns\n        if encoder_embed:\n            self.embedding = nn.Linear(num_columns, d_model)\n        else:\n            self.embedding = nn.Embedding(max_sequence_length, d_model)\n        self.position_encoder = PositionalEncoding(d_model, max_sequence_length)\n        self.dropout = nn.Dropout(p=0.1)\n        \n    def forward(self, x):\n        x = self.embedding(x)\n        pos = self.position_encoder().to(device)\n        x = self.dropout(x + pos)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:47.419175Z","iopub.execute_input":"2023-07-10T10:00:47.420235Z","iopub.status.idle":"2023-07-10T10:00:47.428567Z","shell.execute_reply.started":"2023-07-10T10:00:47.420204Z","shell.execute_reply":"2023-07-10T10:00:47.427548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LayerNormalization(nn.Module):\n    def __init__(self, parameters_shape, eps=1e-5):\n        super().__init__()\n        self.parameters_shape=parameters_shape\n        self.eps=eps\n        self.gamma = nn.Parameter(torch.ones(parameters_shape))\n        self.beta =  nn.Parameter(torch.zeros(parameters_shape))\n\n    def forward(self, inputs):\n        dims = [-(i + 1) for i in range(len(self.parameters_shape))]\n        mean = inputs.mean(dim=dims, keepdim=True)\n        var = ((inputs - mean) ** 2).mean(dim=dims, keepdim=True)\n        std = (var + self.eps).sqrt()\n        y = (inputs - mean) / std\n        out = self.gamma * y + self.beta\n        return out\n\n\nclass PositionwiseFeedForward(nn.Module):\n    def __init__(self, d_model, hidden, drop_prob=0.1):\n        super(PositionwiseFeedForward, self).__init__()\n        self.linear1 = nn.Linear(d_model, hidden)\n        self.linear2 = nn.Linear(hidden, d_model)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(p=drop_prob)\n\n    def forward(self, x):\n        x = self.linear1(x)\n        x = self.relu(x)\n        x = self.dropout(x)\n        x = self.linear2(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:47.430015Z","iopub.execute_input":"2023-07-10T10:00:47.430506Z","iopub.status.idle":"2023-07-10T10:00:47.443709Z","shell.execute_reply.started":"2023-07-10T10:00:47.430406Z","shell.execute_reply":"2023-07-10T10:00:47.44289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scaled_dot_product(q, k, v, mask=None):\n    d_k = q.size()[-1]\n    scaled = torch.matmul(q, k.transpose(-1, -2)) / torch.sqrt(torch.tensor(d_k, dtype=torch.float))\n    if mask is not None:\n        scaled = scaled.permute(1, 0, 2, 3) + mask\n        scaled = scaled.permute(1, 0, 2, 3)\n    attention = F.softmax(scaled, dim=-1)\n    values = torch.matmul(attention, v)\n    return values, attention\n\nclass MultiHeadAttention(nn.Module):\n    def __init__(self, d_model, num_heads):\n        super().__init__()\n        self.d_model = d_model\n        self.num_heads = num_heads\n        self.head_dim = d_model // num_heads\n        self.qkv_layer = nn.Linear(d_model , 3 * d_model)\n        self.linear_layer = nn.Linear(d_model, d_model)\n\n    def forward(self, x, mask):\n        batch_size, sequence_length, d_model = x.size()\n        qkv = self.qkv_layer(x)\n        qkv = qkv.reshape(batch_size, sequence_length, self.num_heads, 3 * self.head_dim)\n        qkv = qkv.permute(0, 2, 1, 3)\n        q, k, v = qkv.chunk(3, dim=-1)\n        values, attention = scaled_dot_product(q, k, v, mask)\n        values = values.permute(0, 2, 1, 3).reshape(batch_size, sequence_length, self.num_heads * self.head_dim)\n        out = self.linear_layer(values)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:47.447303Z","iopub.execute_input":"2023-07-10T10:00:47.447632Z","iopub.status.idle":"2023-07-10T10:00:47.460279Z","shell.execute_reply.started":"2023-07-10T10:00:47.447607Z","shell.execute_reply":"2023-07-10T10:00:47.458016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MultiHeadCrossAttention(nn.Module):\n    def __init__(self, d_model, num_heads):\n        super().__init__()\n        self.d_model = d_model\n        self.num_heads = num_heads\n        self.head_dim = d_model // num_heads\n        self.kv_layer = nn.Linear(d_model , 2 * d_model)\n        self.q_layer = nn.Linear(d_model , d_model)\n        self.linear_layer = nn.Linear(d_model, d_model)\n\n    def forward(self, x, y, mask):\n        batch_size, sequence_length, d_model = x.size() # in practice, this is the same for both languages...so we can technically combine with normal attention\n        kv = self.kv_layer(x)\n        q = self.q_layer(y)\n        kv = kv.reshape(batch_size, sequence_length, self.num_heads, 2 * self.head_dim)\n        q = q.reshape(batch_size, sequence_length, self.num_heads, self.head_dim)\n        kv = kv.permute(0, 2, 1, 3)\n        q = q.permute(0, 2, 1, 3)\n        k, v = kv.chunk(2, dim=-1)\n        values, attention = scaled_dot_product(q, k, v, mask) # We don't need the mask for cross attention, removing in outer function!\n        values = values.permute(0, 2, 1, 3).reshape(batch_size, sequence_length, d_model)\n        out = self.linear_layer(values)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:47.462198Z","iopub.execute_input":"2023-07-10T10:00:47.462656Z","iopub.status.idle":"2023-07-10T10:00:47.474111Z","shell.execute_reply.started":"2023-07-10T10:00:47.462618Z","shell.execute_reply":"2023-07-10T10:00:47.47322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderLayer(nn.Module):\n    def __init__(self, d_model, ffn_hidden, num_heads, drop_prob):\n        super(EncoderLayer, self).__init__()\n        self.attention = MultiHeadAttention(d_model=d_model, num_heads=num_heads)\n        self.norm1 = LayerNormalization(parameters_shape=[d_model])\n        self.dropout1 = nn.Dropout(p=drop_prob)\n        self.ffn = PositionwiseFeedForward(d_model=d_model, hidden=ffn_hidden, drop_prob=drop_prob)\n        self.norm2 = LayerNormalization(parameters_shape=[d_model])\n        self.dropout2 = nn.Dropout(p=drop_prob)\n\n    def forward(self, x, self_attention_mask):\n        residual_x = x.clone()\n        x = self.attention(x, mask=self_attention_mask)\n        x = self.dropout1(x)\n        x = self.norm1(x + residual_x)\n        residual_x = x.clone()\n        x = self.ffn(x)\n        x = self.dropout2(x)\n        x = self.norm2(x + residual_x)\n        return x\n    \nclass SequentialEncoder(nn.Sequential):\n    def forward(self, *inputs):\n        x, self_attention_mask  = inputs\n        for module in self._modules.values():\n            x = module(x, self_attention_mask)\n        return x\n        \nclass Encoder(nn.Module):\n    def __init__(self, d_model, ffn_hidden, num_heads, drop_prob, num_layers, max_sequence_length, num_columns, encoder_embed=True):\n        super().__init__()\n        self.encoder_embedding = CustomEmbedding(max_sequence_length, num_columns, d_model, encoder_embed=True)\n        self.layers = SequentialEncoder(*[EncoderLayer(d_model, ffn_hidden, num_heads, drop_prob)\n                                      for _ in range(num_layers)])\n    \n    def forward(self, x, self_attention_mask):\n        x = self.encoder_embedding(x)\n        x = self.layers(x, self_attention_mask)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:47.476746Z","iopub.execute_input":"2023-07-10T10:00:47.477381Z","iopub.status.idle":"2023-07-10T10:00:47.489999Z","shell.execute_reply.started":"2023-07-10T10:00:47.477351Z","shell.execute_reply":"2023-07-10T10:00:47.489273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DecoderLayer(nn.Module):\n    def __init__(self, d_model, ffn_hidden, num_heads, drop_prob):\n        super(DecoderLayer, self).__init__()\n        self.self_attention = MultiHeadAttention(d_model=d_model, num_heads=num_heads)\n        self.layer_norm1 = LayerNormalization(parameters_shape=[d_model])\n        self.dropout1 = nn.Dropout(p=drop_prob)\n\n        self.encoder_decoder_attention = MultiHeadCrossAttention(d_model=d_model, num_heads=num_heads)\n        self.layer_norm2 = LayerNormalization(parameters_shape=[d_model])\n        self.dropout2 = nn.Dropout(p=drop_prob)\n\n        self.ffn = PositionwiseFeedForward(d_model=d_model, hidden=ffn_hidden, drop_prob=drop_prob)\n        self.layer_norm3 = LayerNormalization(parameters_shape=[d_model])\n        self.dropout3 = nn.Dropout(p=drop_prob)\n\n    def forward(self, x, y, self_attention_mask, cross_attention_mask):\n        _y = y.clone()\n        y = self.self_attention(y, mask=self_attention_mask)\n        y = self.dropout1(y)\n        y = self.layer_norm1(y + _y)\n\n        _y = y.clone()\n        y = self.encoder_decoder_attention(x, y, mask=cross_attention_mask)\n        y = self.dropout2(y)\n        y = self.layer_norm2(y + _y)\n\n        _y = y.clone()\n        y = self.ffn(y)\n        y = self.dropout3(y)\n        y = self.layer_norm3(y + _y)\n        return y\n\n\nclass SequentialDecoder(nn.Sequential):\n    def forward(self, *inputs):\n        x, y, self_attention_mask, cross_attention_mask = inputs\n        for module in self._modules.values():\n            y = module(x, y, self_attention_mask, cross_attention_mask)\n        return y\n\nclass Decoder(nn.Module):\n    def __init__(self, d_model, ffn_hidden, num_heads, drop_prob, num_layers, max_sequence_length, num_columns, encoder_embed=False):\n        super().__init__()\n        self.decoder_embedding = CustomEmbedding(max_sequence_length, num_columns, d_model, encoder_embed=False)\n        self.layers = SequentialDecoder(*[DecoderLayer(d_model, ffn_hidden, num_heads, drop_prob) for _ in range(num_layers)])\n\n    def forward(self, x, y, self_attention_mask, cross_attention_mask):\n        y = self.decoder_embedding(y)\n        y = self.layers(x, y, self_attention_mask, cross_attention_mask)\n        return y","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:47.491263Z","iopub.execute_input":"2023-07-10T10:00:47.49189Z","iopub.status.idle":"2023-07-10T10:00:47.507052Z","shell.execute_reply.started":"2023-07-10T10:00:47.491858Z","shell.execute_reply":"2023-07-10T10:00:47.505974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Transformer(nn.Module):\n    def __init__(self,d_model,\n                          ffn_hidden,\n                          num_heads,\n                          drop_prob,\n                          num_layers,\n                          max_sequence_length,\n                          num_columns,\n                          phrase_vocab_size):\n        super().__init__()\n        self.encoder = Encoder(d_model, ffn_hidden, num_heads, drop_prob, num_layers, max_sequence_length, num_columns, encoder_embed=True)\n        self.decoder = Decoder(d_model, ffn_hidden, num_heads, drop_prob, num_layers, max_sequence_length, num_columns, encoder_embed=False)\n        self.linear = nn.Linear(d_model, phrase_vocab_size)\n        self.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n    def forward(self,\n                x,\n                y,\n                encoder_self_attention_mask=None,\n                decoder_self_attention_mask=None,\n                decoder_cross_attention_mask=None): # x, y are batch of sentences\n        x = self.encoder(x, encoder_self_attention_mask)\n        out = self.decoder(x, y, decoder_self_attention_mask, decoder_cross_attention_mask)\n        out = self.linear(out)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:00:47.508236Z","iopub.execute_input":"2023-07-10T10:00:47.508904Z","iopub.status.idle":"2023-07-10T10:00:47.520948Z","shell.execute_reply.started":"2023-07-10T10:00:47.508872Z","shell.execute_reply":"2023-07-10T10:00:47.52026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformer = Transformer(d_model,ffn_hidden,\n                          num_heads,\n                          drop_prob,\n                          num_layers,\n                          MAX_SEQUENCE_LENGTH,\n                         num_columns,\n                        phrase_vocab_size)\ndisplay(transformer)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T15:27:57.275236Z","iopub.execute_input":"2023-07-10T15:27:57.275633Z","iopub.status.idle":"2023-07-10T15:27:57.386282Z","shell.execute_reply.started":"2023-07-10T15:27:57.275602Z","shell.execute_reply":"2023-07-10T15:27:57.38523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss(ignore_index=char2ord[PADDING_TOKEN],\n                              reduction='none')\n\n# When computing the loss, we are ignoring cases when the label is the padding token\nfor params in transformer.parameters():\n    if params.dim() > 1:\n        nn.init.xavier_uniform_(params)\n\noptimizer = torch.optim.Adam(transformer.parameters(), lr=learning_rate)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T15:27:57.52511Z","iopub.execute_input":"2023-07-10T15:27:57.526026Z","iopub.status.idle":"2023-07-10T15:27:57.631671Z","shell.execute_reply.started":"2023-07-10T15:27:57.525981Z","shell.execute_reply":"2023-07-10T15:27:57.630701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(model, dataloader, criterion, optimizer, device):\n    steps = len(dataloader)\n    total_train_epoch_loss = 0\n    total_train_epoch_acc = 0\n    transformer.train()\n    for batch_idx, batch in tqdm(enumerate(dataloader)):\n        frame = batch['frame']\n        input_phrase = batch['input_phrase']\n        targets = batch['target'].view(-1)\n        encoder_self_attention_mask = batch['encoder_self_attention_mask']\n        decoder_self_attention_mask = batch['decoder_self_attention_mask']\n        decoder_cross_attention_mask = batch['decoder_cross_attention_mask']\n        output = model(frame, input_phrase, encoder_self_attention_mask, decoder_self_attention_mask, decoder_cross_attention_mask)\n\n        batch_loss = criterion(output.view(-1, phrase_vocab_size).to(device), targets.to(device)).to(device)\n        valid_indices = torch.where(targets == char2ord[PADDING_TOKEN], False, True)\n        batch_loss = batch_loss.sum() / valid_indices.sum()\n\n        optimizer.zero_grad()\n        # Backpropagation\n        batch_loss.backward() # Gradients calculation\n        optimizer.step() # Updation of weights\n        batch_acc = (output.argmax(dim=2) == batch['target']).sum() / valid_indices.sum()\n\n        total_train_epoch_loss += batch_loss\n        total_train_epoch_acc += batch_acc\n\n        #total_train_samples+= len(batch['target'].view(-1))\n\n        print(f'Batch train Accuracy: {batch_acc} | Batch train Loss: {batch_loss} | Batch:{batch_idx}')\n    train_epoch_acc = total_train_epoch_acc / steps\n    train_epoch_loss = total_train_epoch_loss / steps\n    return train_epoch_acc, train_epoch_loss","metadata":{"execution":{"iopub.status.busy":"2023-07-10T15:27:57.786087Z","iopub.execute_input":"2023-07-10T15:27:57.786452Z","iopub.status.idle":"2023-07-10T15:27:57.797103Z","shell.execute_reply.started":"2023-07-10T15:27:57.786421Z","shell.execute_reply":"2023-07-10T15:27:57.796018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_model(model, dataloader, device):\n    steps = len(dataloader)\n    total_val_epoch_loss = 0\n    total_val_epoch_acc = 0\n    total_val_samples = 0\n    transformer.eval()\n    with torch.no_grad():\n        for batch_idx, batch in tqdm(enumerate(dataloader)):\n            frame = batch['frame']\n            input_phrase = batch['input_phrase']\n            targets = batch['target'].view(-1)\n            encoder_self_attention_mask = batch['encoder_self_attention_mask']\n            decoder_self_attention_mask = batch['decoder_self_attention_mask']\n            decoder_cross_attention_mask = batch['decoder_cross_attention_mask']\n            output = model(frame, input_phrase, encoder_self_attention_mask, decoder_self_attention_mask, decoder_cross_attention_mask)\n\n            batch_loss = criterion(output.view(-1, phrase_vocab_size).to(device), targets.to(device)).to(device)\n            valid_indices = torch.where(targets == char2ord[PADDING_TOKEN], False, True)\n            batch_loss = batch_loss.sum() / valid_indices.sum()\n\n            batch_acc = (output.argmax(dim=2) == batch['target']).sum() / valid_indices.sum()\n\n            total_val_epoch_loss += batch_loss\n            total_val_epoch_acc += batch_acc\n    \n        #total_val_samples+= len(batch['target'].view(-1))\n\n            print(f'Batch val Accuracy: {batch_acc} | Batch val Loss: {batch_loss} | Batch:{batch_idx}')\n    val_epoch_acc = total_val_epoch_acc / steps\n    val_epoch_loss = total_val_epoch_loss / steps\n    return val_epoch_acc, val_epoch_loss","metadata":{"execution":{"iopub.status.busy":"2023-07-10T15:27:58.049269Z","iopub.execute_input":"2023-07-10T15:27:58.049663Z","iopub.status.idle":"2023-07-10T15:27:58.059816Z","shell.execute_reply.started":"2023-07-10T15:27:58.049631Z","shell.execute_reply":"2023-07-10T15:27:58.058834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_checkpoint (state, filename=\"/kaggle/working/my_checkpoint.pth.tar\"):\n    print(\"=> Saving checkpoint\")\n    torch.save(state, filename)\n    \ndef load_checkpoint (model, optimizer, checkpoint):\n    print(\"=> Loading checkpoint\")\n    model.load_state_dict(checkpoint['state_dict'])\n    optimizer.load_state_dict(checkpoint['optimizer'])\n\nif load_model:\n    load_checkpoint (transformer, optimizer, torch.load(\"/kaggle/working/my_checkpoint.pth.tar\"))","metadata":{"execution":{"iopub.status.busy":"2023-07-10T15:27:58.41987Z","iopub.execute_input":"2023-07-10T15:27:58.420251Z","iopub.status.idle":"2023-07-10T15:27:58.427078Z","shell.execute_reply.started":"2023-07-10T15:27:58.420219Z","shell.execute_reply":"2023-07-10T15:27:58.425728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformer.to(device)\nkf = KFold(n_splits=k)\n# Iterate over the folds\nfor fold, (train_index, val_index) in enumerate(kf.split(dataset)):\n    print(f\"Training on fold {fold+1}/{k}...\")\n\n    # Create data loaders for training and validation\n    train_dataset = Subset(dataset, train_index)\n    val_dataset = Subset(dataset, val_index)\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE)\n    for epoch in range(NUM_EPOCHS):\n        print(f'Epoch {epoch + 1}/{NUM_EPOCHS}')\n        \n        checkpoint = {'state_dict' : transformer.state_dict(), 'optimizer': optimizer.state_dict()}\n        save_checkpoint (checkpoint)\n        train_epoch_acc, train_epoch_loss = train_one_epoch(transformer, train_loader, criterion, optimizer, device)\n        print('-----------------------------------------------------------------')\n        print()\n        print(f'Epoch train Accuracy:{train_epoch_acc} | Epoch train Loss:{train_epoch_loss}')\n        \n        val_epoch_acc, val_epoch_loss = eval_model(transformer, val_loader, device)\n        print(f'Epoch val Accuracy:{val_epoch_acc} | Epoch val Loss:{val_epoch_loss}| Epoch:{epoch + 1}')\n        print()\n        print('-----------------------------------------------------------------')","metadata":{"execution":{"iopub.status.busy":"2023-07-10T15:27:58.791679Z","iopub.execute_input":"2023-07-10T15:27:58.79204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.rmtree('/kaggle/working')","metadata":{"execution":{"iopub.status.busy":"2023-07-10T15:27:48.228204Z","iopub.execute_input":"2023-07-10T15:27:48.228581Z","iopub.status.idle":"2023-07-10T15:27:48.32648Z","shell.execute_reply.started":"2023-07-10T15:27:48.228543Z","shell.execute_reply":"2023-07-10T15:27:48.325018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}