{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## In this notebook, we practice with T5 embeddings and a CNN ","metadata":{}},{"cell_type":"code","source":"MAIN_DIR = \"/kaggle/input/cafa-5-protein-function-prediction\"\n\n# UTILITARIES\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport time\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\n\n# TORCH MODULES FOR METRICS COMPUTATION :\nimport torch\nfrom torch.utils.data import Dataset\nfrom torch import nn\nfrom torch.utils.data import random_split\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom torchmetrics.classification import MultilabelF1Score\nfrom torchmetrics.classification import MultilabelAccuracy\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.loggers import WandbLogger\n\n# WANDB FOR LIGHTNING :\nimport wandb\n\n# FILES VISUALIZATION\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-10T16:23:37.048416Z","iopub.execute_input":"2023-07-10T16:23:37.048959Z","iopub.status.idle":"2023-07-10T16:23:51.395018Z","shell.execute_reply.started":"2023-07-10T16:23:37.048933Z","shell.execute_reply":"2023-07-10T16:23:51.393209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n    train_sequences_path = MAIN_DIR  + \"/Train/train_sequences.fasta\"\n    train_labels_path = MAIN_DIR + \"/Train/train_terms.tsv\"\n    test_sequences_path = MAIN_DIR + \"/Test (Targets)/testsuperset.fasta\"\n    \n    # Change this value for different number of target labels\n    num_labels = 20\n    n_epochs = 5\n    batch_size = 128\n    lr = 0.001\n    \n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:24:58.435876Z","iopub.execute_input":"2023-07-10T16:24:58.436232Z","iopub.status.idle":"2023-07-10T16:24:58.450477Z","shell.execute_reply.started":"2023-07-10T16:24:58.436203Z","shell.execute_reply":"2023-07-10T16:24:58.449506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(config.device)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:25:05.886053Z","iopub.execute_input":"2023-07-10T16:25:05.886519Z","iopub.status.idle":"2023-07-10T16:25:05.89239Z","shell.execute_reply.started":"2023-07-10T16:25:05.886483Z","shell.execute_reply":"2023-07-10T16:25:05.891342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Directories for the different embedding vectors : \nembeds_map = {\n    \"T5\" : \"t5embeds\",\n}\n\n# Length of the different embedding vectors :\nembeds_dim = {\n    \"T5\" : 1024,\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:25:41.978418Z","iopub.execute_input":"2023-07-10T16:25:41.978774Z","iopub.status.idle":"2023-07-10T16:25:41.98407Z","shell.execute_reply.started":"2023-07-10T16:25:41.978748Z","shell.execute_reply":"2023-07-10T16:25:41.982985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make the label table for trainning","metadata":{}},{"cell_type":"code","source":"trainTerms = pd.read_csv(\"/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\",sep=\"\\t\")\nprint(trainTerms.shape)\ndisplay(trainTerms.head(2))\nvec_freqCount = (trainTerms['term'].value_counts())\nprint(vec_freqCount )","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:26:09.615107Z","iopub.execute_input":"2023-07-10T16:26:09.615484Z","iopub.status.idle":"2023-07-10T16:26:14.048651Z","shell.execute_reply.started":"2023-07-10T16:26:09.615455Z","shell.execute_reply":"2023-07-10T16:26:14.047683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_to_consider = list(vec_freqCount.index[:config.num_labels] )\nprint('num_labels:', len(labels_to_consider), 'First 10:', labels_to_consider[:10] ) ","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:26:35.828582Z","iopub.execute_input":"2023-07-10T16:26:35.829432Z","iopub.status.idle":"2023-07-10T16:26:35.834595Z","shell.execute_reply.started":"2023-07-10T16:26:35.829397Z","shell.execute_reply":"2023-07-10T16:26:35.833636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fn = '/kaggle/input/t5embeds/train_ids.npy'\nvec_train_protein_ids = np.load(fn)\nprint(vec_train_protein_ids.shape)\nvec_train_protein_ids","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:27:00.914642Z","iopub.execute_input":"2023-07-10T16:27:00.915219Z","iopub.status.idle":"2023-07-10T16:27:00.966115Z","shell.execute_reply.started":"2023-07-10T16:27:00.915186Z","shell.execute_reply":"2023-07-10T16:27:00.965069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size = 142246 # len(X)\nY = np.zeros( (train_size ,config.num_labels) )\nprint(Y.shape)\n\nseries_train_protein_ids = pd.Series(vec_train_protein_ids ) # \n\ntrainTerms_smaller = trainTerms[ trainTerms['term'].isin( labels_to_consider ) ] # to speed-up the next step \nprint( trainTerms_smaller.shape)\n\nfor i in range(Y.shape[1]):\n    m = trainTerms_smaller['term'] ==  labels_to_consider[i]\n#     m.sum()\n    Y[:,i] =  series_train_protein_ids.isin(  set(trainTerms_smaller[m]['EntryID'] ) ).astype(float )\n    if (i % 10) == 0: \n        print(i, m.sum())\nY","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:29:53.092863Z","iopub.execute_input":"2023-07-10T16:29:53.093707Z","iopub.status.idle":"2023-07-10T16:29:58.283963Z","shell.execute_reply.started":"2023-07-10T16:29:53.093674Z","shell.execute_reply":"2023-07-10T16:29:58.283035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-06T19:47:21.637072Z","iopub.execute_input":"2023-07-06T19:47:21.637547Z","iopub.status.idle":"2023-07-06T19:47:21.644636Z","shell.execute_reply.started":"2023-07-06T19:47:21.637516Z","shell.execute_reply":"2023-07-06T19:47:21.643587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_to_save = \"/kaggle/working/targets_20.npy\"","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:39:38.164392Z","iopub.execute_input":"2023-07-10T16:39:38.164847Z","iopub.status.idle":"2023-07-10T16:39:38.170484Z","shell.execute_reply.started":"2023-07-10T16:39:38.164813Z","shell.execute_reply":"2023-07-10T16:39:38.169245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#np.save(y_to_save, Y)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:39:42.264804Z","iopub.execute_input":"2023-07-10T16:39:42.265165Z","iopub.status.idle":"2023-07-10T16:39:42.288568Z","shell.execute_reply.started":"2023-07-10T16:39:42.265137Z","shell.execute_reply":"2023-07-10T16:39:42.287632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare the training data","metadata":{}},{"cell_type":"code","source":"class ProteinSequenceDataset(Dataset):\n    \n    def __init__(self, datatype):\n        super(ProteinSequenceDataset).__init__()\n        self.datatype = datatype\n        \n        embeds = np.load(\"/kaggle/input/\"+embeds_map['T5']+\"/\"+datatype+\"_embeds.npy\")\n        ids = np.load(\"/kaggle/input/\"+embeds_map['T5']+\"/\"+datatype+\"_ids.npy\")\n            \n        embeds_list = []\n        for l in range(embeds.shape[0]):\n            embeds_list.append(embeds[l,:])\n        self.df = pd.DataFrame(data={\"EntryID\": ids, \"embed\" : embeds_list})\n        \n        if datatype==\"train\":\n            np_labels = Y\n            df_labels = pd.DataFrame(self.df['EntryID'])\n            df_labels['labels_vect']=[row for row in np_labels]\n            self.df = self.df.merge(df_labels, on=\"EntryID\")\n            \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        embed = torch.tensor(self.df.iloc[index][\"embed\"] , dtype = torch.float32)\n        if self.datatype==\"train\":\n            targets = torch.tensor(self.df.iloc[index][\"labels_vect\"], dtype = torch.float32)\n            return embed, targets\n        if self.datatype==\"test\":\n            id = self.df.iloc[index][\"EntryID\"]\n            return embed, id\n        ","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:40:51.057758Z","iopub.execute_input":"2023-07-10T16:40:51.058145Z","iopub.status.idle":"2023-07-10T16:40:51.070435Z","shell.execute_reply.started":"2023-07-10T16:40:51.058118Z","shell.execute_reply":"2023-07-10T16:40:51.067562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling and Training","metadata":{}},{"cell_type":"code","source":"class CNN1D(nn.Module):\n    def __init__(self, input_dim, num_classes):\n        super(CNN1D, self).__init__()\n        # (batch_size, channels, embed_size)\n        self.conv1 = nn.Conv1d(in_channels=1, out_channels=3, kernel_size=3, dilation=1, padding=1, stride=1)\n        # (batch_size, 3, embed_size)\n        self.pool1 = nn.MaxPool1d(kernel_size=2, stride=2)\n        # (batch_size, 3, embed_size/2 = 512)\n        self.conv2 = nn.Conv1d(in_channels=3, out_channels=8, kernel_size=3, dilation=1, padding=1, stride=1)\n        # (batch_size, 8, embed_size/2 = 512)\n        self.pool2 = nn.MaxPool1d(kernel_size=2, stride=2)\n        # (batch_size, 8, embed_size/4 = 256)\n        self.fc1 = nn.Linear(in_features=int(8 * input_dim/4), out_features=128)\n        self.fc2 = nn.Linear(in_features=128, out_features=num_classes)\n\n    def forward(self, x):\n        x = x.reshape(x.shape[0], 1, x.shape[1])\n        x = self.pool1(nn.functional.relu(self.conv1(x)))\n        x = self.pool2(nn.functional.relu(self.conv2(x)))\n        x = torch.flatten(x, 1)\n        x = nn.functional.relu(self.fc1(x))\n        x = self.fc2(x)\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:49:08.609853Z","iopub.execute_input":"2023-07-10T16:49:08.610217Z","iopub.status.idle":"2023-07-10T16:49:08.621524Z","shell.execute_reply.started":"2023-07-10T16:49:08.610188Z","shell.execute_reply":"2023-07-10T16:49:08.619927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(train_size=0.9):\n    \n    train_dataset = ProteinSequenceDataset(datatype=\"train\")\n    \n    train_set, val_set = random_split(train_dataset, lengths = [int(len(train_dataset)*train_size), len(train_dataset)-int(len(train_dataset)*train_size)])\n    train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=config.batch_size, shuffle=True)\n    val_dataloader = torch.utils.data.DataLoader(val_set, batch_size=config.batch_size, shuffle=True)\n\n    model = CNN1D(input_dim=embeds_dim['T5'], num_classes=config.num_labels).to(config.device)\n\n    optimizer = torch.optim.Adam(model.parameters(), lr = config.lr)\n    scheduler = ReduceLROnPlateau(optimizer, factor=0.1, patience=1)\n    CrossEntropy = torch.nn.CrossEntropyLoss()\n    f1_score = MultilabelF1Score(num_labels=config.num_labels).to(config.device)\n    n_epochs = config.n_epochs\n\n    print(\"BEGIN TRAINING...\")\n    train_loss_history=[]\n    val_loss_history=[]\n    \n    train_f1score_history=[]\n    val_f1score_history=[]\n    for epoch in range(n_epochs):\n        print(\"EPOCH \", epoch+1)\n        ## TRAIN PHASE :\n        losses = []\n        scores = []\n        for embed, targets in tqdm(train_dataloader):\n            embed, targets = embed.to(config.device), targets.to(config.device)\n            optimizer.zero_grad()\n            preds = model(embed)\n            loss= CrossEntropy(preds, targets)\n            score=f1_score(preds, targets)\n            losses.append(loss.item()) \n            scores.append(score.item())\n            loss.backward()\n            optimizer.step()\n        avg_loss = np.mean(losses)\n        avg_score = np.mean(scores)\n        print(\"Running Average TRAIN Loss : \", avg_loss)\n        print(\"Running Average TRAIN F1-Score : \", avg_score)\n        train_loss_history.append(avg_loss)\n        train_f1score_history.append(avg_score)\n        \n        ## VALIDATION PHASE : \n        losses = []\n        scores = []\n        for embed, targets in val_dataloader:\n            embed, targets = embed.to(config.device), targets.to(config.device)\n            preds = model(embed)\n            loss= CrossEntropy(preds, targets)\n            score=f1_score(preds, targets)\n            losses.append(loss.item())\n            scores.append(score.item())\n        avg_loss = np.mean(losses)\n        avg_score = np.mean(scores)\n        print(\"Running Average VAL Loss : \", avg_loss)\n        print(\"Running Average VAL F1-Score : \", avg_score)\n        val_loss_history.append(avg_loss)\n        val_f1score_history.append(avg_score)\n        \n        scheduler.step(avg_loss)\n        print(\"\\n\")\n        \n    print(\"TRAINING FINISHED\")\n    print(\"FINAL TRAINING SCORE : \", train_f1score_history[-1])\n    print(\"FINAL VALIDATION SCORE : \", val_f1score_history[-1])\n    \n    losses_history = {\"train\" : train_loss_history, \"val\" : val_loss_history}\n    scores_history = {\"train\" : train_f1score_history, \"val\" : val_f1score_history}\n    \n    return model, losses_history, scores_history","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:49:18.860857Z","iopub.execute_input":"2023-07-10T16:49:18.861219Z","iopub.status.idle":"2023-07-10T16:49:18.880229Z","shell.execute_reply.started":"2023-07-10T16:49:18.861187Z","shell.execute_reply":"2023-07-10T16:49:18.878811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t5_model, t5_losses, t5_scores = train_model()","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:49:33.505971Z","iopub.execute_input":"2023-07-10T16:49:33.506328Z","iopub.status.idle":"2023-07-10T16:51:56.27429Z","shell.execute_reply.started":"2023-07-10T16:49:33.506298Z","shell.execute_reply":"2023-07-10T16:51:56.273196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 4))\n#plt.plot(ems2_losses[\"val\"], label = \"EMS2\")\nplt.plot(t5_losses[\"val\"], label = \"T5\")\n#plt.plot(protbert_losses[\"val\"], label = \"ProtBERT\") \nplt.title(\"Validation Losses for # Vector Embeddings\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Average Loss\")\nplt.legend()\nplt.show()\n\nplt.figure(figsize = (10, 4))\n#plt.plot(ems2_scores[\"val\"], label = \"EMS2\")\nplt.plot(t5_scores[\"val\"], label = \"T5\")\n#plt.plot(protbert_scores[\"val\"], label = \"ProtBERT\")\nplt.title(\"Validation F1-Scores for # Vector Embeddings\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Average F1-Score\")\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:53:42.538659Z","iopub.execute_input":"2023-07-10T16:53:42.539063Z","iopub.status.idle":"2023-07-10T16:53:43.167313Z","shell.execute_reply.started":"2023-07-10T16:53:42.539035Z","shell.execute_reply":"2023-07-10T16:53:43.166268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction","metadata":{}},{"cell_type":"code","source":"def predict():\n    \n    test_dataset = ProteinSequenceDataset(datatype=\"test\")\n    test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False)\n    \n    model = t5_model\n        \n    model.eval()\n    \n    labels = pd.read_csv(config.train_labels_path, sep = \"\\t\")\n    top_terms = labels.groupby(\"term\")[\"EntryID\"].count().sort_values(ascending=False)\n    labels_names = top_terms[:config.num_labels].index.values\n    print(\"GENERATE PREDICTION FOR TEST SET...\")\n\n    ids_ = np.empty(shape=(len(test_dataloader)*config.num_labels,), dtype=object)\n    go_terms_ = np.empty(shape=(len(test_dataloader)*config.num_labels,), dtype=object)\n    confs_ = np.empty(shape=(len(test_dataloader)*config.num_labels,), dtype=np.float32)\n\n    for i, (embed, id) in tqdm(enumerate(test_dataloader), total=len(test_dataloader)):\n        embed = embed.to(config.device)\n        confs_[i*config.num_labels:(i+1)*config.num_labels] = torch.nn.functional.sigmoid(model(embed)).squeeze().detach().cpu().numpy()\n        ids_[i*config.num_labels:(i+1)*config.num_labels] = id[0]\n        go_terms_[i*config.num_labels:(i+1)*config.num_labels] = labels_names\n\n    submission_df = pd.DataFrame(data={\"Id\" : ids_, \"GO term\" : go_terms_, \"Confidence\" : confs_})\n    print(\"PREDICTIONS DONE\")\n    return submission_df","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:54:03.251757Z","iopub.execute_input":"2023-07-10T16:54:03.252123Z","iopub.status.idle":"2023-07-10T16:54:03.264141Z","shell.execute_reply.started":"2023-07-10T16:54:03.252095Z","shell.execute_reply":"2023-07-10T16:54:03.262957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = predict()","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:54:12.467819Z","iopub.execute_input":"2023-07-10T16:54:12.468426Z","iopub.status.idle":"2023-07-10T16:56:30.036308Z","shell.execute_reply.started":"2023-07-10T16:54:12.468386Z","shell.execute_reply":"2023-07-10T16:56:30.035398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(submission_df)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T20:09:42.959356Z","iopub.execute_input":"2023-07-06T20:09:42.959727Z","iopub.status.idle":"2023-07-06T20:09:42.967172Z","shell.execute_reply.started":"2023-07-06T20:09:42.959696Z","shell.execute_reply":"2023-07-06T20:09:42.966194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df[['Id', 'GO term', 'Confidence']].to_csv('/kaggle/working/submission.tsv', sep='\\t', header=False, index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T17:00:03.874294Z","iopub.execute_input":"2023-07-10T17:00:03.875331Z","iopub.status.idle":"2023-07-10T17:00:14.735882Z","shell.execute_reply.started":"2023-07-10T17:00:03.875288Z","shell.execute_reply":"2023-07-10T17:00:14.734914Z"},"trusted":true},"execution_count":null,"outputs":[]}]}