{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n","metadata":{"papermill":{"duration":0.008308,"end_time":"2023-05-09T08:30:09.105539","exception":false,"start_time":"2023-05-09T08:30:09.097231","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Imports","metadata":{"papermill":{"duration":0.009086,"end_time":"2023-05-09T08:30:09.140473","exception":false,"start_time":"2023-05-09T08:30:09.131387","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom keras.models import Sequential\nfrom keras.layers import Dense, BatchNormalization, Dropout\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nimport progressbar","metadata":{"papermill":{"duration":9.85331,"end_time":"2023-05-09T08:30:19.002985","exception":false,"start_time":"2023-05-09T08:30:09.149675","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:52:41.403072Z","iopub.execute_input":"2023-06-28T18:52:41.403469Z","iopub.status.idle":"2023-06-28T18:52:49.87561Z","shell.execute_reply.started":"2023-06-28T18:52:41.403439Z","shell.execute_reply":"2023-06-28T18:52:49.874479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the Dataset","metadata":{"papermill":{"duration":0.008429,"end_time":"2023-05-09T08:30:19.047756","exception":false,"start_time":"2023-05-09T08:30:19.039327","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_terms = pd.read_csv(\"/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\",sep=\"\\t\")\ntrain_terms.head()","metadata":{"papermill":{"duration":3.69155,"end_time":"2023-05-09T08:30:22.766144","exception":false,"start_time":"2023-05-09T08:30:19.074594","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:56:01.789648Z","iopub.execute_input":"2023-06-28T18:56:01.790081Z","iopub.status.idle":"2023-06-28T18:56:05.828352Z","shell.execute_reply.started":"2023-06-28T18:56:01.790048Z","shell.execute_reply":"2023-06-28T18:56:05.826945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading the protein embeddings","metadata":{}},{"cell_type":"code","source":"train_protein_ids = np.load('/kaggle/input/t5embeds/train_ids.npy')\ntrain_protein_ids[:5]","metadata":{"papermill":{"duration":0.067806,"end_time":"2023-05-09T08:30:22.944355","exception":false,"start_time":"2023-05-09T08:30:22.876549","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:56:05.830526Z","iopub.execute_input":"2023-06-28T18:56:05.830905Z","iopub.status.idle":"2023-06-28T18:56:05.84493Z","shell.execute_reply.started":"2023-06-28T18:56:05.830871Z","shell.execute_reply":"2023-06-28T18:56:05.843993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_embeddings = np.load('/kaggle/input/t5embeds/train_embeds.npy')\n\ncolumn_num = train_embeddings.shape[1]\ntrain_df = pd.DataFrame(train_embeddings, columns = [\"Column_\" + str(i) for i in range(1, column_num+1)])\ntrain_df.head()","metadata":{"papermill":{"duration":9.719957,"end_time":"2023-05-09T08:30:32.741095","exception":false,"start_time":"2023-05-09T08:30:23.021138","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:56:05.846332Z","iopub.execute_input":"2023-06-28T18:56:05.847439Z","iopub.status.idle":"2023-06-28T18:56:06.320523Z","shell.execute_reply.started":"2023-06-28T18:56:05.847393Z","shell.execute_reply":"2023-06-28T18:56:06.319186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare the dataset","metadata":{"papermill":{"duration":0.009208,"end_time":"2023-05-09T08:30:32.825978","exception":false,"start_time":"2023-05-09T08:30:32.81677","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plot_df = train_terms['term'].value_counts().iloc[:100]\n\nfigure, axis = plt.subplots(1, 1, figsize=(12, 6))\n\nbp = sns.barplot(ax=axis, x=np.array(plot_df.index), y=plot_df.values)\nbp.set_xticklabels(bp.get_xticklabels(), rotation=90, size = 6)\naxis.set_title('Top 100 frequent GO term IDs')\nbp.set_xlabel(\"GO term IDs\", fontsize = 12)\nbp.set_ylabel(\"Count\", fontsize = 12)\nplt.show()","metadata":{"papermill":{"duration":1.592489,"end_time":"2023-05-09T08:30:34.465912","exception":false,"start_time":"2023-05-09T08:30:32.873423","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:56:06.323541Z","iopub.execute_input":"2023-06-28T18:56:06.324379Z","iopub.status.idle":"2023-06-28T18:56:09.373732Z","shell.execute_reply.started":"2023-06-28T18:56:06.324329Z","shell.execute_reply":"2023-06-28T18:56:09.372564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_of_labels = 1500\n\nlabels = train_terms['term'].value_counts().index[:num_of_labels].tolist()","metadata":{"papermill":{"duration":0.523976,"end_time":"2023-05-09T08:30:35.021974","exception":false,"start_time":"2023-05-09T08:30:34.497998","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:56:09.376233Z","iopub.execute_input":"2023-06-28T18:56:09.37699Z","iopub.status.idle":"2023-06-28T18:56:10.402767Z","shell.execute_reply.started":"2023-06-28T18:56:09.376942Z","shell.execute_reply":"2023-06-28T18:56:10.400997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_terms_updated = train_terms.loc[train_terms['term'].isin(labels)]","metadata":{"papermill":{"duration":0.668657,"end_time":"2023-05-09T08:30:35.720953","exception":false,"start_time":"2023-05-09T08:30:35.052296","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:56:10.404305Z","iopub.execute_input":"2023-06-28T18:56:10.404713Z","iopub.status.idle":"2023-06-28T18:56:11.187829Z","shell.execute_reply.started":"2023-06-28T18:56:10.404679Z","shell.execute_reply":"2023-06-28T18:56:11.186498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pie_df = train_terms_updated['aspect'].value_counts()\npalette_color = sns.color_palette('bright')\nplt.pie(pie_df.values, labels=np.array(pie_df.index), colors=palette_color, autopct='%.0f%%')\nplt.show()","metadata":{"papermill":{"duration":0.419624,"end_time":"2023-05-09T08:30:36.171193","exception":false,"start_time":"2023-05-09T08:30:35.751569","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:56:11.190068Z","iopub.execute_input":"2023-06-28T18:56:11.190711Z","iopub.status.idle":"2023-06-28T18:56:12.048252Z","shell.execute_reply.started":"2023-06-28T18:56:11.190678Z","shell.execute_reply":"2023-06-28T18:56:12.046697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bar = progressbar.ProgressBar(maxval=num_of_labels, \\\n    widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])\n\ntrain_size = train_protein_ids.shape[0] # len(X)\ntrain_labels = np.zeros((train_size ,num_of_labels))\n\nseries_train_protein_ids = pd.Series(train_protein_ids)\n\nfor i in range(num_of_labels):\n    n_train_terms = train_terms_updated[train_terms_updated['term'] ==  labels[i]]\n    \n    label_related_proteins = n_train_terms['EntryID'].unique()\n    \n    train_labels[:,i] =  series_train_protein_ids.isin(label_related_proteins).astype(float)\n    \n    bar.update(i+1)\n\nbar.finish()\n\nlabels_df = pd.DataFrame(data = train_labels, columns = labels)\nlabels_df.head()","metadata":{"papermill":{"duration":495.729474,"end_time":"2023-05-09T08:38:51.951408","exception":false,"start_time":"2023-05-09T08:30:36.221934","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:14:15.704317Z","iopub.execute_input":"2023-06-28T18:14:15.704706Z","iopub.status.idle":"2023-06-28T18:33:18.479877Z","shell.execute_reply.started":"2023-06-28T18:14:15.704674Z","shell.execute_reply":"2023-06-28T18:33:18.478792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape[1]","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:33:18.485764Z","iopub.execute_input":"2023-06-28T18:33:18.488852Z","iopub.status.idle":"2023-06-28T18:33:18.501158Z","shell.execute_reply.started":"2023-06-28T18:33:18.488802Z","shell.execute_reply":"2023-06-28T18:33:18.500056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building NN","metadata":{"papermill":{"duration":0.010523,"end_time":"2023-05-09T08:38:52.052433","exception":false,"start_time":"2023-05-09T08:38:52.04191","status":"completed"},"tags":[]}},{"cell_type":"code","source":"INPUT_SHAPE = [1024]\nBATCH_SIZE = 5120\n\ndef Build_Model() -> Sequential:\n    model = Sequential()\n\n    model.add(Dense(64, activation='relu', input_shape=INPUT_SHAPE))\n    model.add(BatchNormalization())\n\n    model.add(Dense(128, activation='relu'))\n    model.add(BatchNormalization())\n\n    model.add(Dense(256, activation='relu'))\n    model.add(BatchNormalization())\n\n    model.add(Dense(512, activation='relu'))\n    model.add(Dropout(0.25))\n\n    model.add(Dense(512, activation='relu'))\n    model.add(Dropout(0.25))\n\n    model.add(Dense(1500, activation='sigmoid'))\n\n    return model","metadata":{"papermill":{"duration":128.96621,"end_time":"2023-05-09T08:41:01.029422","exception":false,"start_time":"2023-05-09T08:38:52.063212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:56:16.988284Z","iopub.execute_input":"2023-06-28T18:56:16.988708Z","iopub.status.idle":"2023-06-28T18:56:16.99808Z","shell.execute_reply.started":"2023-06-28T18:56:16.988677Z","shell.execute_reply":"2023-06-28T18:56:16.996875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Build_Model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:56:17.198188Z","iopub.execute_input":"2023-06-28T18:56:17.198899Z","iopub.status.idle":"2023-06-28T18:56:17.399366Z","shell.execute_reply.started":"2023-06-28T18:56:17.198853Z","shell.execute_reply":"2023-06-28T18:56:17.398097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['binary_accuracy', 'AUC'])\n\ncheckpoint = ModelCheckpoint(\"protein_weights.h5\", monitor='val_loss', verbose=1, save_best_only=True, mode='min', save_weights_only = True)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=2, verbose=1, mode='auto', min_delta=0.0001, cooldown=5)\n\nearly_stop = EarlyStopping(monitor='val_loss', mode='min', patience=20, restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:33:18.786947Z","iopub.execute_input":"2023-06-28T18:33:18.78733Z","iopub.status.idle":"2023-06-28T18:33:18.841627Z","shell.execute_reply.started":"2023-06-28T18:33:18.787299Z","shell.execute_reply":"2023-06-28T18:33:18.840529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"history = model.fit(\n    train_df, labels_df,\n    batch_size=BATCH_SIZE,\n    epochs=1000,\n    validation_split=0.2,\n    callbacks=[checkpoint, early_stop, reduceLROnPlat]\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:33:18.843505Z","iopub.execute_input":"2023-06-28T18:33:18.844813Z","iopub.status.idle":"2023-06-28T18:36:48.055149Z","shell.execute_reply.started":"2023-06-28T18:33:18.844727Z","shell.execute_reply":"2023-06-28T18:36:48.053996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot the model's loss and accuracy for each epoch","metadata":{"papermill":{"duration":0.019782,"end_time":"2023-05-09T08:41:01.06997","exception":false,"start_time":"2023-05-09T08:41:01.050188","status":"completed"},"tags":[]}},{"cell_type":"code","source":"acc = history.history['binary_accuracy']\nval_acc = history.history['val_binary_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nplt.figure(figsize=(10, 16))\nplt.rcParams['figure.figsize'] = [16, 9]\nplt.rcParams['font.size'] = 14\nplt.rcParams['axes.grid'] = True\nplt.rcParams['figure.facecolor'] = 'white'\nplt.subplot(2, 1, 1)\nplt.plot(acc, label='Training Accuracy')\nplt.plot(val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.ylabel('Accuracy')\nplt.title(f'Training and Validation Accuracy. \\nTrain Accuracy: {str(acc[-1])}\\nValidation Accuracy: {str(val_acc[-1])}')\nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":0.647806,"end_time":"2023-05-09T08:41:01.737745","exception":false,"start_time":"2023-05-09T08:41:01.089939","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:37:01.174541Z","iopub.execute_input":"2023-06-28T18:37:01.175278Z","iopub.status.idle":"2023-06-28T18:37:01.627882Z","shell.execute_reply.started":"2023-06-28T18:37:01.175233Z","shell.execute_reply":"2023-06-28T18:37:01.626832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplot(2, 1, 2)\nplt.plot(loss, label='Training Loss')\nplt.plot(val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.ylabel('Cross Entropy')\nplt.title(f'Training and Validation Loss. \\nTrain Loss: {str(loss[-1])}\\nValidation Loss: {str(val_loss[-1])}')\nplt.xlabel('epoch')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:37:55.663807Z","iopub.execute_input":"2023-06-28T18:37:55.664261Z","iopub.status.idle":"2023-06-28T18:37:56.108419Z","shell.execute_reply.started":"2023-06-28T18:37:55.664227Z","shell.execute_reply":"2023-06-28T18:37:56.107384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.021665,"end_time":"2023-05-09T08:41:01.780867","exception":false,"start_time":"2023-05-09T08:41:01.759202","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_embeddings = np.load('/kaggle/input/t5embeds/test_embeds.npy')\n\ncolumn_num = test_embeddings.shape[1]\ntest_df = pd.DataFrame(test_embeddings, columns = [\"Column_\" + str(i) for i in range(1, column_num+1)])\ntest_df.head()","metadata":{"papermill":{"duration":10.290827,"end_time":"2023-05-09T08:41:12.134919","exception":false,"start_time":"2023-05-09T08:41:01.844092","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:56:26.027984Z","iopub.execute_input":"2023-06-28T18:56:26.028403Z","iopub.status.idle":"2023-06-28T18:56:26.502875Z","shell.execute_reply.started":"2023-06-28T18:56:26.028372Z","shell.execute_reply":"2023-06-28T18:56:26.501984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights('/kaggle/working/protein_weights.h5')","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:56:30.058406Z","iopub.execute_input":"2023-06-28T18:56:30.058814Z","iopub.status.idle":"2023-06-28T18:56:30.155202Z","shell.execute_reply.started":"2023-06-28T18:56:30.058784Z","shell.execute_reply":"2023-06-28T18:56:30.154194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions =  model.predict(test_df)","metadata":{"papermill":{"duration":663.907351,"end_time":"2023-05-09T08:52:16.178461","exception":false,"start_time":"2023-05-09T08:41:12.27111","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T18:56:31.218096Z","iopub.execute_input":"2023-06-28T18:56:31.218517Z","iopub.status.idle":"2023-06-28T18:56:54.209555Z","shell.execute_reply.started":"2023-06-28T18:56:31.218481Z","shell.execute_reply":"2023-06-28T18:56:54.208664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = pd.DataFrame(columns = ['Protein Id', 'GO Term Id','Prediction'])\ntest_protein_ids = np.load('/kaggle/input/t5embeds/test_ids.npy')\nl = []\nfor k in list(test_protein_ids):\n    l += [ k] * predictions.shape[1]   \n\ndf_submission['Protein Id'] = l\ndf_submission['GO Term Id'] = labels * predictions.shape[0]\ndf_submission['Prediction'] = predictions.ravel()\ndf_submission.to_csv(\"submission.tsv\",header=False, index=False, sep=\"\\t\")","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:57:00.988187Z","iopub.execute_input":"2023-06-28T18:57:00.989193Z","iopub.status.idle":"2023-06-28T19:14:50.609612Z","shell.execute_reply.started":"2023-06-28T18:57:00.989151Z","shell.execute_reply":"2023-06-28T19:14:50.606695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission","metadata":{"papermill":{"duration":0.063739,"end_time":"2023-05-09T08:52:16.292974","exception":false,"start_time":"2023-05-09T08:52:16.229235","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-28T19:15:09.452263Z","iopub.execute_input":"2023-06-28T19:15:09.452708Z","iopub.status.idle":"2023-06-28T19:15:09.470624Z","shell.execute_reply.started":"2023-06-28T19:15:09.452676Z","shell.execute_reply":"2023-06-28T19:15:09.46908Z"},"trusted":true},"execution_count":null,"outputs":[]}]}