{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"###IMPORTANDO BIBLIOTECAS UTILIZADAS ##\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Required for progressbar widget\nimport progressbar\n\nprint(\"TensorFlow v\" + tf.__version__)\nprint(\"Numpy v\" + np.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-23T15:38:23.3182Z","iopub.execute_input":"2023-05-23T15:38:23.318712Z","iopub.status.idle":"2023-05-23T15:38:23.332637Z","shell.execute_reply.started":"2023-05-23T15:38:23.31867Z","shell.execute_reply":"2023-05-23T15:38:23.331358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_terms = pd.read_csv(\"/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\",sep=\"\\t\")\nprint(\"do banco de dados possui o tamanho de: \")\nprint(train_terms.shape)\ntrain_terms.head()\n\n# entry id = é o que precisamos prever ( sequencia de aminoacidos)\n\n\n#aspect - é uma variavel pra auxiliar , uma caracteristica da proteina \n#GO = gene ontology é um indicativo \n\n\n# temos 5363863 proteinas \n\n# a quantidade de valores unicos dentro desse dataframe \nproteinas_unicas = train_terms['EntryID'].unique()\nprint()\nprint(\"desses valores são unicos: \")\nprint(proteinas_unicas.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T15:38:23.33442Z","iopub.execute_input":"2023-05-23T15:38:23.335634Z","iopub.status.idle":"2023-05-23T15:38:26.936452Z","shell.execute_reply.started":"2023-05-23T15:38:23.335584Z","shell.execute_reply":"2023-05-23T15:38:26.935305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explorando um datasheet que possui só as variaveis unicas, exportado para popoar trabalho\ntrain_protein_ids = np.load('/kaggle/input/t5embeds/train_ids.npy')\nprint(train_protein_ids.shape)\n\ntrain_embeddings = np.load('/kaggle/input/t5embeds/train_embeds.npy')\n\n#um codigo que já vem integrado para que a gente possa usar no codigo \ncolumn_num = train_embeddings.shape[1]\ntrain_df = pd.DataFrame(train_embeddings, columns = [\"Column_\" + str(i) for i in range(1, column_num+1)])\n#comprovando que tem o memsmo tamanho do arquivo original \nprint(train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T15:38:26.938517Z","iopub.execute_input":"2023-05-23T15:38:26.939363Z","iopub.status.idle":"2023-05-23T15:38:27.649049Z","shell.execute_reply.started":"2023-05-23T15:38:26.93932Z","shell.execute_reply":"2023-05-23T15:38:27.647823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T15:38:27.652282Z","iopub.execute_input":"2023-05-23T15:38:27.652668Z","iopub.status.idle":"2023-05-23T15:38:27.680889Z","shell.execute_reply.started":"2023-05-23T15:38:27.652635Z","shell.execute_reply":"2023-05-23T15:38:27.679891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#PREPARANDO O DATASHEET\n\ntrainTerms = pd.read_csv(\"/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\",sep=\"\\t\")\nprint(trainTerms.shape)\ndisplay(trainTerms.head(2))\nvec_freqCount = (trainTerms['term'].value_counts())\nprint(vec_freqCount )\n\n\n\n\nfn = '/kaggle/input/t5embeds/train_ids.npy'\nvec_train_protein_ids = np.load(fn)\nprint(vec_train_protein_ids.shape)\nvec_train_protein_ids","metadata":{"execution":{"iopub.status.busy":"2023-05-23T15:38:27.682377Z","iopub.execute_input":"2023-05-23T15:38:27.683014Z","iopub.status.idle":"2023-05-23T15:38:31.821859Z","shell.execute_reply.started":"2023-05-23T15:38:27.682979Z","shell.execute_reply":"2023-05-23T15:38:31.820824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select first 1500 values for plotting\nplot_df = train_terms['term'].value_counts().iloc[:100]\n\nfigure, axis = plt.subplots(1, 1, figsize=(12, 6))\n\nbp = sns.barplot(ax=axis, x=np.array(plot_df.index), y=plot_df.values)\nbp.set_xticklabels(bp.get_xticklabels(), rotation=90, size = 6)\n#os itens mais frequentes que apareceram no banco de dados \naxis.set_title('Top 100 frequent GO term IDs')\nbp.set_xlabel(\"GO term IDs\", fontsize = 12)\nbp.set_ylabel(\"Count\", fontsize = 12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T15:38:31.823696Z","iopub.execute_input":"2023-05-23T15:38:31.824575Z","iopub.status.idle":"2023-05-23T15:38:34.082185Z","shell.execute_reply.started":"2023-05-23T15:38:31.824531Z","shell.execute_reply":"2023-05-23T15:38:34.080864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nnum_of_labels = 1500 #(saber qual Id GO (quantidade de saida que estamos prevendo ))\n\n## pega os valores decressentes dos termos \nlabels = train_terms['term'].value_counts().index[:num_of_labels].tolist()\ntrain_terms_updated = train_terms.loc[train_terms['term'].isin(labels)]\n# i.e, train_size x num_of_labels (142246 x 1500)\ntrain_size = train_protein_ids.shape[0] # len(X)\ntrain_labels = np.zeros((train_size ,num_of_labels))\n\n\n #convertendo o vetor que estvamos usando em uma serie do pandas\nseries_train_protein_ids = pd.Series(train_protein_ids)\n\n#  #convertendo o vetor que estvamos usando em uma serie do pandas so que dessa vez é para as colunas \nfor i in range(num_of_labels):\n    # For each label, fetch the corresponding train_terms data\n    n_train_terms = train_terms_updated[train_terms_updated['term'] ==  labels[i]]\n    \n    \n    # ciou-se uma coluna só com os valores unicos \n   \n    label_related_proteins = n_train_terms['EntryID'].unique()\n    \n    #falando que coloca 1 se a proteina aparcee e 0 se ela não aparece \n    train_labels[:,i] =  series_train_protein_ids.isin(label_related_proteins).astype(float)\n            #train_labels = np.zeros((train_size ,num_of_labels))\n            #vai aparecer uma base em binario se aparece a proteina ou não , \n            # é tipo um encording \n    #seria tipo o nosso x \n\n\n\n# converte o Y pro dataframe também ( que vai ser os rotolos das proteinas)\nlabels_df = pd.DataFrame(data = train_labels, columns = labels)\nprint(labels_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T15:38:34.084084Z","iopub.execute_input":"2023-05-23T15:38:34.084594Z","iopub.status.idle":"2023-05-23T15:59:00.955472Z","shell.execute_reply.started":"2023-05-23T15:38:34.08455Z","shell.execute_reply":"2023-05-23T15:59:00.954104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tamanho da entrada \nINPUT_SHAPE = [train_df.shape[1]]\n# ce dá um tamanho para o sistema trabalhar \nBATCH_SIZE = 5120\n\n\n# criação dos neurinios da rede neural com o tensor flow \nmodel = tf.keras.Sequential([\n    tf.keras.layers.BatchNormalization(input_shape=INPUT_SHAPE),    \n    tf.keras.layers.Dense(units=512, activation='relu'),\n    tf.keras.layers.Dense(units=512, activation='relu'),\n    tf.keras.layers.Dense(units=512, activation='relu'),\n    tf.keras.layers.Dense(units=num_of_labels,activation='sigmoid')\n])\n\n\n# Compile model\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n    loss='binary_crossentropy',\n    metrics=['binary_accuracy', tf.keras.metrics.AUC()],\n)\n\nhistory = model.fit(\n    train_df, labels_df,\n    batch_size=BATCH_SIZE,\n    epochs=5\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T15:59:00.957004Z","iopub.execute_input":"2023-05-23T15:59:00.957322Z","iopub.status.idle":"2023-05-23T16:01:20.421021Z","shell.execute_reply.started":"2023-05-23T15:59:00.957296Z","shell.execute_reply":"2023-05-23T16:01:20.419906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_embeddings = np.load('/kaggle/input/t5embeds/test_embeds.npy')\n\n# Convert test_embeddings to dataframe\ncolumn_num = test_embeddings.shape[1]\ntest_df = pd.DataFrame(test_embeddings, columns = [\"Column_\" + str(i) for i in range(1, column_num+1)])\nprint(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:01:20.422723Z","iopub.execute_input":"2023-05-23T16:01:20.423591Z","iopub.status.idle":"2023-05-23T16:01:32.6423Z","shell.execute_reply.started":"2023-05-23T16:01:20.423553Z","shell.execute_reply":"2023-05-23T16:01:32.641146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions =  model.predict(test_df)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:01:32.645806Z","iopub.execute_input":"2023-05-23T16:01:32.647205Z","iopub.status.idle":"2023-05-23T16:02:15.064209Z","shell.execute_reply.started":"2023-05-23T16:01:32.647154Z","shell.execute_reply":"2023-05-23T16:02:15.063336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n## Parte para a subimissão \n\ndf_submission = pd.DataFrame(columns = ['Protein Id', 'GO Term Id','Prediction'])\ntest_protein_ids = np.load('/kaggle/input/t5embeds/test_ids.npy')\nl = []\nfor k in list(test_protein_ids):\n    l += [ k] * predictions.shape[1]   \n\ndf_submission['Protein Id'] = l\ndf_submission['GO Term Id'] = labels * predictions.shape[0]\ndf_submission['Prediction'] = predictions.ravel()\ndf_submission.to_csv(\"submission.tsv\",header=False, index=False, sep=\"\\t\")","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:02:15.065994Z","iopub.execute_input":"2023-05-23T16:02:15.066556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}