{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.8.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":41875,"databundleVersionId":5521661,"sourceType":"competition"},{"sourceId":5607816,"sourceType":"datasetVersion","datasetId":3225525},{"sourceId":5807641,"sourceType":"datasetVersion","datasetId":3211581},{"sourceId":6216319,"sourceType":"datasetVersion","datasetId":3569938},{"sourceId":6222023,"sourceType":"datasetVersion","datasetId":3573469},{"sourceId":6222633,"sourceType":"datasetVersion","datasetId":3573825},{"sourceId":6245049,"sourceType":"datasetVersion","datasetId":3588444},{"sourceId":6245839,"sourceType":"datasetVersion","datasetId":3588867},{"sourceId":6250098,"sourceType":"datasetVersion","datasetId":3591759},{"sourceId":6250347,"sourceType":"datasetVersion","datasetId":3591902},{"sourceId":6259750,"sourceType":"datasetVersion","datasetId":3597827},{"sourceId":6260035,"sourceType":"datasetVersion","datasetId":3597993},{"sourceId":6278630,"sourceType":"datasetVersion","datasetId":3609819},{"sourceId":6279337,"sourceType":"datasetVersion","datasetId":3610244},{"sourceId":6289275,"sourceType":"datasetVersion","datasetId":3616886},{"sourceId":6289936,"sourceType":"datasetVersion","datasetId":3617278},{"sourceId":6290805,"sourceType":"datasetVersion","datasetId":3617855},{"sourceId":6306586,"sourceType":"datasetVersion","datasetId":3627525},{"sourceId":6308391,"sourceType":"datasetVersion","datasetId":3629477},{"sourceId":140384618,"sourceType":"kernelVersion"}],"dockerImageVersionId":30528,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# config\n- recommendation, this notebook runs fine with cpu!","metadata":{}},{"cell_type":"code","source":"#only include go terms with certain probabaility\ninclude_go_terms_with_probability = 0.000001\n\nsq_len = 1000000\n\n# use the same value as in the other notebooks\nmost_common_gene_ontologies = 5000\n\n#path to the finetuned nn\nmodel_path = \"/kaggle/input/training-esm2-3b/new_esm2_3b_fine_tuning_5000_4096_batch_vec_1\"\nmodel_path\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:01:51.734393Z","iopub.execute_input":"2023-08-19T21:01:51.735125Z","iopub.status.idle":"2023-08-19T21:01:51.752703Z","shell.execute_reply.started":"2023-08-19T21:01:51.735087Z","shell.execute_reply":"2023-08-19T21:01:51.751715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade pip","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:01:54.333657Z","iopub.execute_input":"2023-08-19T21:01:54.334556Z","iopub.status.idle":"2023-08-19T21:01:59.498553Z","shell.execute_reply.started":"2023-08-19T21:01:54.33452Z","shell.execute_reply":"2023-08-19T21:01:59.4972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install biopython\n!pip install numpy\n!pip install pip install torchmetrics","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:02:11.220079Z","iopub.execute_input":"2023-08-19T21:02:11.220457Z","iopub.status.idle":"2023-08-19T21:02:25.978026Z","shell.execute_reply.started":"2023-08-19T21:02:11.220423Z","shell.execute_reply":"2023-08-19T21:02:25.976802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n\nimport torch\n#from Bio import SeqIO\n#import plotly.graph_objects as go\n#from collections import Counter\n\nimport sys\n\n#from transformers import BertModel, BertTokenizer\nimport re\n\nfrom timeit import default_timer as timer\nfrom datetime import timedelta\nimport time \nfrom datetime import datetime\n\n#from glob import glob\n\n#from scipy.special import softmax\nnp.set_printoptions(precision=5)\n\ntorch.set_printoptions(threshold=5)\ntorch.manual_seed(0)\n\npd.set_option('display.float_format', lambda x: '%.20f' % x)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-19T21:03:19.73613Z","iopub.execute_input":"2023-08-19T21:03:19.737037Z","iopub.status.idle":"2023-08-19T21:03:19.775789Z","shell.execute_reply.started":"2023-08-19T21:03:19.736992Z","shell.execute_reply":"2023-08-19T21:03:19.775051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev\nimport torch_xla\nimport torch_xla.core.xla_model as xm\n\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:03:23.043103Z","iopub.execute_input":"2023-08-19T21:03:23.043791Z","iopub.status.idle":"2023-08-19T21:03:51.777401Z","shell.execute_reply.started":"2023-08-19T21:03:23.043754Z","shell.execute_reply":"2023-08-19T21:03:51.775979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = xm.xla_device()\ntorch.set_default_tensor_type('torch.FloatTensor')\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:06:14.946743Z","iopub.execute_input":"2023-08-19T21:06:14.947563Z","iopub.status.idle":"2023-08-19T21:06:14.953686Z","shell.execute_reply.started":"2023-08-19T21:06:14.947526Z","shell.execute_reply":"2023-08-19T21:06:14.952785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EasyFeedForward(torch.nn.Module):\n    \n        def __init__(self, input_size, hidden_size, output_size):\n            super(EasyFeedForward, self).__init__()\n            self.layer1 = torch.nn.Linear(input_size, hidden_size)\n            self.activiation1 = torch.nn.ReLU()\n            self.dropout1 = torch.nn.Dropout(0.5)\n            self.layer2 = torch.nn.Linear(hidden_size, hidden_size)\n            self.activiation2 = torch.nn.ReLU()\n            self.dropout2 = torch.nn.Dropout(0.5)\n            self.layer3 = torch.nn.Linear(hidden_size, hidden_size)\n            self.activiation3 = torch.nn.ReLU()\n            self.layer4 = torch.nn.Linear(hidden_size, hidden_size)\n            self.activiation4 = torch.nn.ReLU()\n            self.layer5 = torch.nn.Linear(hidden_size, hidden_size)\n            self.activiation5 = torch.nn.ReLU()\n            self.layer6 = torch.nn.Linear(hidden_size, hidden_size)\n            self.activiation6 = torch.nn.ReLU()\n            self.layer7 = torch.nn.Linear(hidden_size, output_size)\n            \n        def forward(self, x):\n            output = self.layer1(x)\n            output = self.activiation1(output)\n            output = self.dropout1(output)\n            output = self.layer2(output)\n            output = self.activiation2(output)\n            output = self.dropout2(output)\n            output = self.layer3(output)\n            output = self.activiation3(output)\n            output = self.layer4(output)\n            output = self.activiation4(output)\n            output = self.layer5(output)\n            output = self.activiation5(output)\n            output = self.layer6(output)\n            output = self.activiation6(output)\n            output = self.layer7(output)\n            return output\n\n#(1) init FeedForwardNet \nnn_input_size  = 2560 #last_hidden \nnn_hidden_size = most_common_gene_ontologies\nnn_output_size = most_common_gene_ontologies #3000     \n        \nff_nn_model = EasyFeedForward(nn_input_size,nn_hidden_size,nn_output_size ).to(device)\n\n# print(\"NeuralNet uses gpu/tpu: \" + str(next(ff_nn_model.parameters()).is_cuda))\n# print(\"\")\nff_nn_model","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:04:04.225093Z","iopub.execute_input":"2023-08-19T21:04:04.225691Z","iopub.status.idle":"2023-08-19T21:04:05.906284Z","shell.execute_reply.started":"2023-08-19T21:04:04.225627Z","shell.execute_reply":"2023-08-19T21:04:05.905416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#read test sequences and predict GOs\ntest_sequences_fasta = \"/kaggle/input/cafa-5-protein-function-prediction/Test (Targets)/testsuperset.fasta\"\ntest_sequences_full = SeqIO.parse(test_sequences_fasta, 'fasta')\ntest_sequences =  np.array([seq.seq for seq in SeqIO.parse(test_sequences_fasta, 'fasta')], dtype=object)\ntest_sequences_entry_id =  np.array([id.id for id in SeqIO.parse(test_sequences_fasta, 'fasta')], dtype=object)\n\nlen(test_sequences)","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:04:08.355732Z","iopub.execute_input":"2023-08-19T21:04:08.356104Z","iopub.status.idle":"2023-08-19T21:04:13.659479Z","shell.execute_reply.started":"2023-08-19T21:04:08.356072Z","shell.execute_reply":"2023-08-19T21:04:13.658661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#this methods will help with finding the most common gene ontologies (GO)\ndef count_elements(lst):\n    counts = {}\n    for element in lst:\n        if element in counts:\n            counts[element] += 1\n        else:\n            counts[element] = 1\n    return counts\n\ndef sort_by_count(counts):\n    sorted_counts = sorted(counts.items(), key=lambda x: x[1], reverse=True)\n    return sorted_counts","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:04:13.661062Z","iopub.execute_input":"2023-08-19T21:04:13.661357Z","iopub.status.idle":"2023-08-19T21:04:13.667637Z","shell.execute_reply.started":"2023-08-19T21:04:13.661331Z","shell.execute_reply":"2023-08-19T21:04:13.666736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#filter unique gene ontologies (GOs) and create one hot encodings (target vecs)\ngeneOntologyTerms = pd.read_csv(\"/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\",sep=\"\\t\")\ngeneOntologyTerms =  geneOntologyTerms.sort_values(by=[\"EntryID\"])\nallGO = geneOntologyTerms[\"term\"]\nuniqueGO = allGO.unique()\nprint(\"all go:    \" + str(len(allGO)))\nprint(\"unique go: \" + str(len(uniqueGO)))\n#uniqueGO.sort()\n\n#find the 500 most frequent GOs\ncommon_go_with_count = sort_by_count(count_elements(allGO))\ncommon_go_with_count = common_go_with_count[:most_common_gene_ontologies]\n\ncommon_go = []\nfor cg in common_go_with_count:\n    common_go.append(cg[0])\n#inplace sorting\ncommon_go.sort()\n#oneHotGo = pd.get_dummies(common_go)","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:04:13.668804Z","iopub.execute_input":"2023-08-19T21:04:13.669116Z","iopub.status.idle":"2023-08-19T21:04:24.622052Z","shell.execute_reply.started":"2023-08-19T21:04:13.669089Z","shell.execute_reply":"2023-08-19T21:04:24.621164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create indexes for GO terms\nindexed_gos = {}\n\n#assign GOs to one hots\nfor i, go in enumerate(common_go):\n    indexed_gos[i] = go\n\nlen(indexed_gos)","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:04:24.623945Z","iopub.execute_input":"2023-08-19T21:04:24.624258Z","iopub.status.idle":"2023-08-19T21:04:24.632544Z","shell.execute_reply.started":"2023-08-19T21:04:24.624232Z","shell.execute_reply":"2023-08-19T21:04:24.631568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#load precomputed last_hidden_states embeddings \ntest_last_hidden_states = np.load(\"/kaggle/input/4637427/test_embeds_esm2_t36_3B_UR50D.npy\",allow_pickle=True)\ntest_eid = np.load(\"/kaggle/input/4637427/test_ids_esm2_t36_3B_UR50D.npy\",allow_pickle=True) ","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:04:24.633815Z","iopub.execute_input":"2023-08-19T21:04:24.634153Z","iopub.status.idle":"2023-08-19T21:04:52.044997Z","shell.execute_reply.started":"2023-08-19T21:04:24.634125Z","shell.execute_reply":"2023-08-19T21:04:52.044133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:12:59.875118Z","iopub.execute_input":"2023-08-19T21:12:59.876195Z","iopub.status.idle":"2023-08-19T21:12:59.881495Z","shell.execute_reply.started":"2023-08-19T21:12:59.876151Z","shell.execute_reply":"2023-08-19T21:12:59.880619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ff_nn_model = torch.load(model_path,map_location=torch.device(device))\nff_nn_model","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:19:47.892004Z","iopub.execute_input":"2023-08-19T21:19:47.892454Z","iopub.status.idle":"2023-08-19T21:19:56.124609Z","shell.execute_reply.started":"2023-08-19T21:19:47.892417Z","shell.execute_reply":"2023-08-19T21:19:56.123521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"e = ff_nn_model(torch.FloatTensor(1,2560).to(device))\n\nlen(e.detach().cpu()[0])","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:19:56.126075Z","iopub.execute_input":"2023-08-19T21:19:56.126395Z","iopub.status.idle":"2023-08-19T21:19:56.357467Z","shell.execute_reply.started":"2023-08-19T21:19:56.126356Z","shell.execute_reply":"2023-08-19T21:19:56.356534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# compute predictions","metadata":{}},{"cell_type":"code","source":"result_entry_id    = []\nresult_go          = []\nresult_probability = []\n\n\nwith torch.no_grad():\n    \n\n    print(datetime.now())\n    for i,entry_id,lhst in zip(range(len(test_eid)),test_eid,test_last_hidden_states):\n        #print(i,entry_id,lhst)\n        \n        #nn output to GO probability\n        l = torch.from_numpy(lhst).float().to(device)\n        predi = torch.nn.functional.sigmoid(ff_nn_model(l))\n        predi_num = predi.cpu().numpy()\n        #print(len(predi_num))\n        \n        #filter everthing out thats below 'include_go_terms_with_probability' (returns indexes of the wanted GO terms)\n        indexes = np.where(predi_num > include_go_terms_with_probability)\n        indexes = indexes[0]\n        #print(indexes)\n        #convert indexes to GO terms \n        for idx in indexes:\n#             print(indexed_gos[idx])\n#             print(predi[idx])\n            \n            result_entry_id.append(entry_id)\n            result_go.append(indexed_gos[idx])\n            result_probability.append(predi[idx].item())\n            \n        if i%10 == 0 and i > 0:\n            print(i,datetime.now())\n            #break\n            \nprint(len(result_go))\n            \nresult_entry_id[0:5], result_go[0:5], result_probability[0:5]","metadata":{"execution":{"iopub.status.busy":"2023-08-19T21:22:30.60256Z","iopub.execute_input":"2023-08-19T21:22:30.60291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save to tsv\ndata = pd.DataFrame({'entry_id': result_entry_id,'go': result_go,'pred': result_probability})\ndata","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[data[\"entry_id\"] == \"W8DXL4\" ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_sorted = data.sort_values([\"entry_id\",\"pred\"],ascending=False)\ndata_sorted","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_sorted[\"subcount\"] = data_sorted.groupby(\"entry_id\").cumcount()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = data_sorted[data_sorted[\"subcount\"] < 1500 ]\nsubmission = submission.drop(columns=['subcount'])\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.tsv',sep='\\t', header=True, index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}