{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":41875,"databundleVersionId":5521661,"sourceType":"competition"},{"sourceId":5607816,"sourceType":"datasetVersion","datasetId":3225525},{"sourceId":5792099,"sourceType":"datasetVersion","datasetId":3327296}],"dockerImageVersionId":30513,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"MAIN_DIR = \"/kaggle/input/cafa-5-protein-function-prediction\"\n\n# UTILITARIES\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport time\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\nfrom sklearn.model_selection import train_test_split\n# TORCH MODULES FOR METRICS COMPUTATION :\nimport torch\nfrom torch.utils import data\nfrom torch.utils.data import Dataset\nfrom torch import nn\nfrom torch.utils.data import random_split\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom torchmetrics.classification import MultilabelF1Score\nfrom torchmetrics.classification import MultilabelAccuracy\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.loggers import WandbLogger\nimport torch.nn.functional as F\n# KERAS\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Embedding, LSTM, SpatialDropout1D\nfrom keras.callbacks import EarlyStopping\nimport tensorflow as tf\n# WANDB FOR LIGHTNING :\nimport wandb\n\n# FILES VISUALIZATION\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nfrom transformers import BertModel, BertTokenizer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-04T15:27:29.337929Z","iopub.execute_input":"2023-07-04T15:27:29.33853Z","iopub.status.idle":"2023-07-04T15:27:50.905463Z","shell.execute_reply.started":"2023-07-04T15:27:29.338486Z","shell.execute_reply":"2023-07-04T15:27:50.904357Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:98: UserWarning: unable to load libtensorflow_io_plugins.so: unable to open file: libtensorflow_io_plugins.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so']\ncaused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so: undefined symbol: _ZN3tsl6StatusC1EN10tensorflow5error4CodeESt17basic_string_viewIcSt11char_traitsIcEENS_14SourceLocationE']\n  warnings.warn(f\"unable to load libtensorflow_io_plugins.so: {e}\")\n/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:104: UserWarning: file system plugins are not loaded: unable to open file: libtensorflow_io.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so']\ncaused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so: undefined symbol: _ZTVN10tensorflow13GcsFileSystemE']\n  warnings.warn(f\"file system plugins are not loaded: {e}\")\n","output_type":"stream"},{"name":"stdout","text":"/kaggle/input/train-targets-top500/train_targets_top500.npy\n/kaggle/input/protbert-embeddings-for-cafa5/train_ids.npy\n/kaggle/input/protbert-embeddings-for-cafa5/train_embeddings.npy\n/kaggle/input/protbert-embeddings-for-cafa5/test_ids.npy\n/kaggle/input/protbert-embeddings-for-cafa5/test_embeddings.npy\n/kaggle/input/cafa-5-protein-function-prediction/sample_submission.tsv\n/kaggle/input/cafa-5-protein-function-prediction/IA.txt\n/kaggle/input/cafa-5-protein-function-prediction/Test (Targets)/testsuperset.fasta\n/kaggle/input/cafa-5-protein-function-prediction/Test (Targets)/testsuperset-taxon-list.tsv\n/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\n/kaggle/input/cafa-5-protein-function-prediction/Train/train_sequences.fasta\n/kaggle/input/cafa-5-protein-function-prediction/Train/train_taxonomy.tsv\n/kaggle/input/cafa-5-protein-function-prediction/Train/go-basic.obo\n","output_type":"stream"}]},{"cell_type":"code","source":"class config:\n    train_sequences_path = MAIN_DIR  + \"/Train/train_sequences.fasta\"\n    train_labels_path = MAIN_DIR + \"/Train/train_terms.tsv\"\n    test_sequences_path = MAIN_DIR + \"/Test (Targets)/testsuperset.fasta\"\n    \n    num_labels = 500\n    n_epochs = 5\n    batch_size = 128\n    lr = 0.001\n    MAX_FEATURES = 1024\n    LSTM_UNITS = 256\n    NUM_AUX_TARGETS = 6\n    embed_size = 1024\n    TRAINED_MODEL_DIR = 'lstm/'\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:27:59.916566Z","iopub.execute_input":"2023-07-04T15:27:59.91695Z","iopub.status.idle":"2023-07-04T15:27:59.943396Z","shell.execute_reply.started":"2023-07-04T15:27:59.916918Z","shell.execute_reply":"2023-07-04T15:27:59.942316Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"markdown","source":"### LOAD PROBERT EMBEDDINGS","metadata":{}},{"cell_type":"code","source":"train_embeds = np.load('/kaggle/input/protbert-embeddings-for-cafa5/train_embeddings.npy')\ntrain_ids = np.load('/kaggle/input/protbert-embeddings-for-cafa5/train_ids.npy')","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:01.864192Z","iopub.execute_input":"2023-07-04T15:28:01.864592Z","iopub.status.idle":"2023-07-04T15:28:08.718256Z","shell.execute_reply.started":"2023-07-04T15:28:01.864561Z","shell.execute_reply":"2023-07-04T15:28:08.717188Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"train_embeds.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:08.720243Z","iopub.execute_input":"2023-07-04T15:28:08.720586Z","iopub.status.idle":"2023-07-04T15:28:08.727355Z","shell.execute_reply.started":"2023-07-04T15:28:08.720559Z","shell.execute_reply":"2023-07-04T15:28:08.72649Z"},"trusted":true},"execution_count":4,"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"(142246, 1024)"},"metadata":{}}]},{"cell_type":"code","source":"test_embeds = np.load('/kaggle/input/protbert-embeddings-for-cafa5/test_embeddings.npy')\ntest_ids = np.load('/kaggle/input/protbert-embeddings-for-cafa5/test_ids.npy')","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:08.729036Z","iopub.execute_input":"2023-07-04T15:28:08.729672Z","iopub.status.idle":"2023-07-04T15:28:15.625048Z","shell.execute_reply.started":"2023-07-04T15:28:08.729639Z","shell.execute_reply":"2023-07-04T15:28:15.623933Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"test_embeds.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:15.630054Z","iopub.execute_input":"2023-07-04T15:28:15.630808Z","iopub.status.idle":"2023-07-04T15:28:15.63857Z","shell.execute_reply.started":"2023-07-04T15:28:15.630771Z","shell.execute_reply":"2023-07-04T15:28:15.637611Z"},"trusted":true},"execution_count":6,"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"(141865, 1024)"},"metadata":{}}]},{"cell_type":"code","source":"IX = np.arange(train_embeds.shape[0])\nIX_train, IX_test, _,_ = train_test_split( IX, IX, train_size=0.5, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:15.639988Z","iopub.execute_input":"2023-07-04T15:28:15.640607Z","iopub.status.idle":"2023-07-04T15:28:15.660495Z","shell.execute_reply.started":"2023-07-04T15:28:15.640573Z","shell.execute_reply":"2023-07-04T15:28:15.659622Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"markdown","source":"### Generating and saving top K labels","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"print(\"GENERATE TARGETS FOR ENTRY IDS (\"+str(config.num_labels)+\" MOST COMMON GO TERMS)\")\nids = np.load(\"/kaggle/input/protbert-embeddings-for-cafa5/train_ids.npy\")\nlabels = pd.read_csv(config.train_labels_path, sep = \"\\t\")\n\ntop_terms = labels.groupby(\"term\")[\"EntryID\"].count().sort_values(ascending=False)\nlabels_names = top_terms[:config.num_labels].index.values\ntrain_labels_sub = labels[(labels.term.isin(labels_names)) & (labels.EntryID.isin(ids))]\nid_labels = train_labels_sub.groupby('EntryID')['term'].apply(list).to_dict()\n\ngo_terms_map = {label: i for i, label in enumerate(labels_names)}\nlabels_matrix = np.empty((len(ids), len(labels_names)))\n\nfor index, id in tqdm(enumerate(ids)):\n    id_gos_list = id_labels[id]\n    temp = [go_terms_map[go] for go in labels_names if go in id_gos_list]\n    labels_matrix[index, temp] = 1\n\nnp.save(\"/kaggle/working/train_targets_top\"+str(config.num_labels)+\".npy\", np.array(labels_matrix))\nprint(\"GENERATION FINISHED!\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(labels_names)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('label-names-top-500.npy',labels_names)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load saved labels","metadata":{}},{"cell_type":"code","source":"Y = np.load('/kaggle/input/train-targets-top500/train_targets_top500.npy')","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:15.661944Z","iopub.execute_input":"2023-07-04T15:28:15.662336Z","iopub.status.idle":"2023-07-04T15:28:22.273538Z","shell.execute_reply.started":"2023-07-04T15:28:15.662303Z","shell.execute_reply":"2023-07-04T15:28:22.2723Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"Y.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:22.275257Z","iopub.execute_input":"2023-07-04T15:28:22.275954Z","iopub.status.idle":"2023-07-04T15:28:22.283474Z","shell.execute_reply.started":"2023-07-04T15:28:22.275918Z","shell.execute_reply":"2023-07-04T15:28:22.282116Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"(142246, 500)"},"metadata":{}}]},{"cell_type":"markdown","source":"### LSTM MODEL","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:22.285143Z","iopub.execute_input":"2023-07-04T15:28:22.286099Z","iopub.status.idle":"2023-07-04T15:28:22.292268Z","shell.execute_reply.started":"2023-07-04T15:28:22.286072Z","shell.execute_reply":"2023-07-04T15:28:22.291223Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"torch.cuda.is_available()","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:22.294183Z","iopub.execute_input":"2023-07-04T15:28:22.294946Z","iopub.status.idle":"2023-07-04T15:28:22.306482Z","shell.execute_reply.started":"2023-07-04T15:28:22.294912Z","shell.execute_reply":"2023-07-04T15:28:22.30537Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"True"},"metadata":{}}]},{"cell_type":"code","source":"x_train_torch = torch.tensor(train_embeds[IX_train,:], dtype=torch.float32).to(device)\nx_val_torch = torch.tensor(train_embeds[IX_test,:], dtype=torch.float32).to(device)\ny_train_torch = torch.tensor(Y[IX_train,:], dtype=torch.float32).to(device)\ny_val_torch = torch.tensor(Y[IX_test,:], dtype=torch.float32).to(device)\n\ntrain_dataset = data.TensorDataset(x_train_torch, y_train_torch)\nval_dataset = data.TensorDataset(x_val_torch, y_val_torch)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:22.309719Z","iopub.execute_input":"2023-07-04T15:28:22.31054Z","iopub.status.idle":"2023-07-04T15:28:31.298471Z","shell.execute_reply.started":"2023-07-04T15:28:22.310507Z","shell.execute_reply":"2023-07-04T15:28:31.297409Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"class SpatialDropout(nn.Dropout2d):\n    def forward(self, x):\n        x = x.unsqueeze(2)\n        x = x.permute(0, 3, 2, 1)\n        x = super(SpatialDropout, self).forward(x)\n        x = x.permute(0, 3, 2, 1)\n        x = x.squeeze(2)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:31.300597Z","iopub.execute_input":"2023-07-04T15:28:31.300972Z","iopub.status.idle":"2023-07-04T15:28:31.307962Z","shell.execute_reply.started":"2023-07-04T15:28:31.300937Z","shell.execute_reply":"2023-07-04T15:28:31.30673Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"class Lstm(nn.Module):\n    def __init__(self):\n        super(Lstm, self).__init__()\n#         embed_size = embedding_matrix.shape[1]\n#         self.embedding = nn.Embedding(config.MAX_FEATURES, embed_size)\n#         self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n#         self.embedding.weight.requires_grad = False\n#         self.embedding_dropout = SpatialDropout(0.3)\n        self.lstm1 = nn.LSTM(config.embed_size, config.LSTM_UNITS, bidirectional=False, batch_first=True)\n        self.lstm2 = nn.LSTM(config.LSTM_UNITS, config.LSTM_UNITS, bidirectional=False, batch_first=True)\n        \n        self.linear1 = nn.Linear(256, 128)\n        self.linear2 = nn.Linear(config.LSTM_UNITS, 128)\n        \n        self.linear_out = nn.Linear(config.LSTM_UNITS, 1)\n        self.linear_aux_out = nn.Linear(config.LSTM_UNITS, config.NUM_AUX_TARGETS)\n    \n    def forward(self, x):\n#         h_embedding = self.embedding(x)\n#         h_embedding = self.embedding_dropout(h_embedding)\n        h_lstm1, _ = self.lstm1(x)\n        h_lstm2, _ = self.lstm2(h_lstm1)\n        \n        # global average pooling\n        avg_pool = torch.mean(h_lstm2, 1)\n        \n        # global max pooling\n        max_pool, _ = torch.max(h_lstm2, 1)\n#         avg_pool = torch.unsqueeze(avg_pool, 0)\n#         max_pool = torch.unsqueeze(max_pool, 0)\n        print(avg_pool.shape)\n        print(max_pool.shape)\n        h_conc = torch.cat((max_pool, avg_pool), 0)\n        h_conc_linear1 = F.relu(self.linear1(h_conc))\n        h_conc_linear2 = F.relu(self.linear2(h_conc))\n        h_conc_linear = torch.cat((h_conc_linear1, h_conc_linear2), 0)\n        print(h_conc_linear.shape)\n        hidden = h_conc + h_conc_linear\n        result = self.linear_out(hidden)\n        aux_result = self.linear_aux_out(hidden)\n        out = torch.cat([result, aux_result], 0)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:39:03.04184Z","iopub.execute_input":"2023-07-04T15:39:03.042266Z","iopub.status.idle":"2023-07-04T15:39:03.056286Z","shell.execute_reply.started":"2023-07-04T15:39:03.042234Z","shell.execute_reply":"2023-07-04T15:39:03.054168Z"},"trusted":true},"execution_count":52,"outputs":[]},{"cell_type":"code","source":"def train_model(model, train, test, model_file, model_name, loss_fn, lr=0.001, batch_size=512, n_epochs=10):\n    param_lrs = [{'params': param, 'lr': lr} for param in model.parameters()]\n    optimizer = torch.optim.Adam(param_lrs, lr=lr)\n    scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda epoch: 0.6 ** epoch)\n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n    training_loss = []\n    validation_loss = []\n    \n    best_loss = float(\"inf\")\n    for epoch in range(n_epochs):\n        start_time = time.time()\n        \n        model.train()\n        avg_loss = 0\n        \n        for data in tqdm(train_loader, disable=False):\n            x_batch = data[:-1]\n            y_batch = data[-1]\n            if model_name != 'attention':\n                y_pred = model(*x_batch)\n            else:\n                \n                y_pred, _ = model(*x_batch, config.MAX_LEN)\n            loss = loss_fn(y_pred, y_batch)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n            \n        training_loss.append(avg_loss)\n        model.eval()\n        _logger.info(f'... Validating {model_name} ... ')\n        avg_val_loss = 0\n        for val_data in tqdm(test_loader, disable=False):\n            x_batch = val_data[:-1]\n            y_batch = val_data[-1]\n            if model_name != 'attention':\n                y_pred = model(*x_batch)\n            else:\n                y_pred, _ = model(*x_batch, config.MAXLEN)\n            \n            val_loss = loss_fn(y_pred, y_batch)\n            avg_val_loss += val_loss.item() / len(test_loader)\n        \n        elapsed_time = time.time() - start_time\n        validation_loss.append(avg_val_loss)\n        if avg_val_loss < best_loss:\n            _logger.info('saving the best model so far')\n            best_loss = avg_val_loss\n            torch.save(model.state_dict(), model_file)\n        _logger.info(\n            f'Epoch {epoch + 1}/{n_epochs}\\t training_loss={avg_loss:.4f} \\t validation_loss={avg_val_loss: 4f} \\t time={elapsed_time:.2f}s')\n        scheduler.step()\n    return training_loss, validation_loss","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:31.328094Z","iopub.execute_input":"2023-07-04T15:28:31.328534Z","iopub.status.idle":"2023-07-04T15:28:31.345782Z","shell.execute_reply.started":"2023-07-04T15:28:31.328501Z","shell.execute_reply":"2023-07-04T15:28:31.344604Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"def custom_loss(data, targets):\n        ''' Define custom loss function for weighted BCE on 'target' column '''\n        bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:, 1:2])(data[:, :1], targets[:, :1])\n        bce_loss_2 = nn.BCEWithLogitsLoss()(data[:, 1:], targets[:, 2:])\n        return (bce_loss_1 * loss_weights) + bce_loss_2","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:28:31.347085Z","iopub.execute_input":"2023-07-04T15:28:31.348678Z","iopub.status.idle":"2023-07-04T15:28:31.358924Z","shell.execute_reply.started":"2023-07-04T15:28:31.348645Z","shell.execute_reply":"2023-07-04T15:28:31.357905Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"lstm_model = Lstm()","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:39:06.161176Z","iopub.execute_input":"2023-07-04T15:39:06.161779Z","iopub.status.idle":"2023-07-04T15:39:06.206582Z","shell.execute_reply.started":"2023-07-04T15:39:06.161731Z","shell.execute_reply":"2023-07-04T15:39:06.205522Z"},"trusted":true},"execution_count":53,"outputs":[]},{"cell_type":"code","source":"lstm_model.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:39:06.36473Z","iopub.execute_input":"2023-07-04T15:39:06.365416Z","iopub.status.idle":"2023-07-04T15:39:06.387587Z","shell.execute_reply.started":"2023-07-04T15:39:06.36537Z","shell.execute_reply":"2023-07-04T15:39:06.385323Z"},"trusted":true},"execution_count":54,"outputs":[{"execution_count":54,"output_type":"execute_result","data":{"text/plain":"Lstm(\n  (lstm1): LSTM(1024, 256, batch_first=True)\n  (lstm2): LSTM(256, 256, batch_first=True)\n  (linear1): Linear(in_features=256, out_features=128, bias=True)\n  (linear2): Linear(in_features=256, out_features=128, bias=True)\n  (linear_out): Linear(in_features=256, out_features=1, bias=True)\n  (linear_aux_out): Linear(in_features=256, out_features=6, bias=True)\n)"},"metadata":{}}]},{"cell_type":"code","source":"model_file = config.TRAINED_MODEL_DIR + '/probert_1024_v1.pt'\ntraining_loss, validation_loss = train_model(lstm_model, train_dataset, val_dataset, model_file, model_name=\"lstm\",\n                                                 n_epochs=1,loss_fn=nn.BCEWithLogitsLoss(), batch_size = config.batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:40:16.335579Z","iopub.execute_input":"2023-07-04T15:40:16.33665Z","iopub.status.idle":"2023-07-04T15:40:17.198575Z","shell.execute_reply.started":"2023-07-04T15:40:16.336611Z","shell.execute_reply":"2023-07-04T15:40:17.196956Z"},"trusted":true},"execution_count":58,"outputs":[{"name":"stderr","text":"  0%|          | 0/556 [00:00<?, ?it/s]\n","output_type":"stream"},{"name":"stdout","text":"torch.Size([128])\ntorch.Size([128])\ntorch.Size([256])\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)","Cell \u001b[0;32mIn[58], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m model_file \u001b[38;5;241m=\u001b[39m config\u001b[38;5;241m.\u001b[39mTRAINED_MODEL_DIR \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/probert_1024_v1.pt\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m----> 2\u001b[0m training_loss, validation_loss \u001b[38;5;241m=\u001b[39m \u001b[43mtrain_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlstm_model\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_dataset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mval_dataset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel_file\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlstm\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      3\u001b[0m \u001b[43m                                                 \u001b[49m\u001b[43mn_epochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43mloss_fn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mBCEWithLogitsLoss\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m)\u001b[49m\n","Cell \u001b[0;32mIn[15], line 25\u001b[0m, in \u001b[0;36mtrain_model\u001b[0;34m(model, train, test, model_file, model_name, loss_fn, lr, batch_size, n_epochs)\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     24\u001b[0m     y_pred, _ \u001b[38;5;241m=\u001b[39m model(\u001b[38;5;241m*\u001b[39mx_batch, config\u001b[38;5;241m.\u001b[39mMAX_LEN)\n\u001b[0;32m---> 25\u001b[0m loss \u001b[38;5;241m=\u001b[39m \u001b[43mloss_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_pred\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_batch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     26\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[1;32m     27\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/modules/loss.py:720\u001b[0m, in \u001b[0;36mBCEWithLogitsLoss.forward\u001b[0;34m(self, input, target)\u001b[0m\n\u001b[1;32m    719\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor, target: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 720\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbinary_cross_entropy_with_logits\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    721\u001b[0m \u001b[43m                                              \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    722\u001b[0m \u001b[43m                                              \u001b[49m\u001b[43mpos_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpos_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    723\u001b[0m \u001b[43m                                              \u001b[49m\u001b[43mreduction\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreduction\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/functional.py:3163\u001b[0m, in \u001b[0;36mbinary_cross_entropy_with_logits\u001b[0;34m(input, target, weight, size_average, reduce, reduction, pos_weight)\u001b[0m\n\u001b[1;32m   3160\u001b[0m     reduction_enum \u001b[38;5;241m=\u001b[39m _Reduction\u001b[38;5;241m.\u001b[39mget_enum(reduction)\n\u001b[1;32m   3162\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (target\u001b[38;5;241m.\u001b[39msize() \u001b[38;5;241m==\u001b[39m \u001b[38;5;28minput\u001b[39m\u001b[38;5;241m.\u001b[39msize()):\n\u001b[0;32m-> 3163\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTarget size (\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m) must be the same as input size (\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(target\u001b[38;5;241m.\u001b[39msize(), \u001b[38;5;28minput\u001b[39m\u001b[38;5;241m.\u001b[39msize()))\n\u001b[1;32m   3165\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mbinary_cross_entropy_with_logits(\u001b[38;5;28minput\u001b[39m, target, weight, pos_weight, reduction_enum)\n","\u001b[0;31mValueError\u001b[0m: Target size (torch.Size([128, 500])) must be the same as input size (torch.Size([7]))"],"ename":"ValueError","evalue":"Target size (torch.Size([128, 500])) must be the same as input size (torch.Size([7]))","output_type":"error"}]},{"cell_type":"markdown","source":"## SIMPLER IMPLEMENTATION :","metadata":{}},{"cell_type":"code","source":"new_shape = (1, train_embeds.shape[0])  # Add a dimension at the beginning\n# X_train = np.reshape(train_embeds, new_shape)\nX_train = np.expand_dims(train_embeds, axis=-1)\nX_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = tf.config.list_physical_devices('GPU')\nif gpus:\n    try:\n        # Enable GPU memory growth\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n    except RuntimeError as e:\n        print(e)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.client import device_lib\nprint(device_lib.list_local_devices())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sess = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(log_device_placement=True))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('/GPU:1'):\n    embed_size = train_embeds.shape[1]\n    model = Sequential()\n    model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))\n    model.add(Dense(1024, activation='relu'))\n    model.add(Dense(512, activation='relu'))\n    model.add(Dense(500, activation='softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n\n    epochs = 5\n    batch_size = 64\n\n    history = model.fit(X_train[IX_train,:], Y[IX_train,:], epochs=epochs, batch_size=batch_size,validation_split=0.1,callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"probert_1024_v1\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_json = model.to_json()\nwith open(\"lstm_probert_1024_v1.json\", \"w\") as json_file:\n    json_file.write(model_json)\n# serialize weights to HDF5\nmodel.save_weights(\"model.h5\")\nprint(\"Saved model to disk\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accr = model.evaluate(X_train[IX_train,:],Y[IX_train,:])\nprint('Test set\\n  Loss: {:0.3f}\\n  Accuracy: {:0.3f}'.format(accr[0],accr[1]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = np.expand_dims(test_embeds, axis=-1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred = model.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(\"lstm_probert_1024_v1.npy\",Y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### USING PYTORCH","metadata":{}},{"cell_type":"code","source":"class LSTM():\n    def __init__(self):\n        super(Lstm, self).__init__()\n        self.lstm1 = nn.LSTM(config.embed_size, config.LSTM_UNITS, bidirectional=False, batch_first=True)\n        self.linear1 = nn.Linear(256, config.LSTM_UNITS)\n        \n        self.linear_out = nn.Linear(config.LSTM_UNITS * 2, 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}