{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":41875,"databundleVersionId":5521661,"sourceType":"competition"},{"sourceId":5792099,"sourceType":"datasetVersion","datasetId":3327296},{"sourceId":5806680,"sourceType":"datasetVersion","datasetId":3335200}],"dockerImageVersionId":30512,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom Bio import SeqIO\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import pad_sequences # padding sequences\nfrom keras.utils import to_categorical\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow import keras\nfrom tensorflow.keras import Input, Model\nfrom tensorflow.keras.layers import Embedding, Bidirectional, Dropout, Dense\nfrom tensorflow.compat.v1.keras.layers import CuDNNLSTM\nfrom tensorflow.keras.regularizers import l2,l1,l1_l2\nfrom keras.callbacks import EarlyStopping\nimport keras_tuner as kt\nfrom sklearn.metrics import accuracy_score\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"codes = ['A', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'K', 'L',\n         'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'V', 'W', 'Y']\ndef create_dict(codes):\n  char_dict = {}\n  for index, val in enumerate(codes):\n    char_dict[val] = index+1\n\n  return char_dict\n\nchar_dict = create_dict(codes)\n\nprint(char_dict)\nprint(\"Dict Length:\", len(char_dict))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sequences = SeqIO.parse(\"/kaggle/input/cafa-5-protein-function-prediction/Train/train_sequences.fasta\", \"fasta\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sequences = SeqIO.parse(\"/kaggle/input/cafa-5-protein-function-prediction/Test (Targets)/testsuperset.fasta\", \"fasta\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_encode = []\nfor seq in train_sequences:\n    row_encode = []\n    for code in seq:\n        row_encode.append(char_dict.get(code, 0))\n    train_encode.append(np.array(row_encode))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_encode = []\nfor seq in test_sequences:\n    row_encode = []\n    for code in seq:\n        row_encode.append(char_dict.get(code, 0))\n    test_encode.append(np.array(row_encode))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_encode[1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_size = int(len(train_encode)/2)\ntrain_data_size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_encode = train_encode[:train_data_size]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make length of all sequences equal 750 by padding smaller sequences and truncating larger ones","metadata":{}},{"cell_type":"code","source":"max_length = 750\ntrain_pad = pad_sequences(train_encode, maxlen=max_length, padding='post', truncating='post')\ntrain_pad.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pad = pad_sequences(test_encode, maxlen=max_length, padding='post', truncating='post')\ntest_pad.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y = np.load('/kaggle/input/xgbdata/Y_1499.npy')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y = Y[:train_data_size,:]\nY.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IX = np.arange(train_pad.shape[0])\nIX_train, IX_test, _,_ = train_test_split( IX, IX, train_size=0.5, random_state=42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y[IX_train,:].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_input = Input(shape=(max_length,))\nemb = Embedding(500, 512, input_length=max_length)(x_input)\nbi_rnn = Bidirectional(CuDNNLSTM(320, kernel_regularizer=l2(0.0001), recurrent_regularizer=l2(0.001), bias_regularizer=l2(0.001)))(emb)\nx = Dropout(0.3)(bi_rnn)\nx = Dense(256,activation='relu')(x)\nx_output = Dense(1499, activation='sigmoid')(x)\n\nmodel1 = Model(inputs=x_input, outputs=x_output)\nmodel1.compile(optimizer=keras.optimizers.Adam(1e-5), loss=tf.nn.sigmoid_cross_entropy_with_logits, metrics=['accuracy'])\n\nmodel1.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 45\nbatch_size = 128","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"es = EarlyStopping(monitor='val_loss', patience=3, verbose=1)\nhistory1 = model1.fit(train_pad[IX_train,:], Y[IX_train,:], epochs=epochs, batch_size=batch_size,validation_split=0.1, callbacks=[es])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred = model1.predict(train_pad[IX_test,:])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_vector(vector):\n    min_val = np.min(vector)\n    max_val = np.max(vector)\n    normalized_vector = (vector - min_val) / (max_val - min_val)\n    return normalized_vector","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def F1_score(y_pred,y_target):\n    y_pred = normalize_vector(y_pred)\n    threshold = 0.5\n    val_preds = np.where(y_pred> threshold, 1, 0)\n    val_labels = y_target\n    tp = np.sum((val_preds == 1) & (val_labels == 1))\n    fp = np.sum((val_preds == 1) & (val_labels == 0))\n    fn = np.sum((val_preds == 0) & (val_labels == 1))\n    if tp+fp == 0:\n        return 0\n    precision = tp / (tp + fp)\n    if tp+fn == 0:\n        return 0\n    recall = tp / (tp + fn)\n    if (precision + recall) == 0:\n        return 0\n    f1_score = 2 * (precision * recall) / (precision + recall)\n\n    return f1_score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"normalize_vector(Y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"F1_score(Y_pred,Y[IX_test,:])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model1.predict(test_pad)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(\"pred.npy\", predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Manual HyperParameter Tuning","metadata":{}},{"cell_type":"code","source":"num_epochs = 10\nbatch_sz = 256\n\nparam_grid = {\n    'lstm_units': [256,512],\n    'dropout_rate': [0.05,0.001],\n#     'kernel_regularizer': [None, l1(0.01), l2(0.01), l1_l2(l1=0.01, l2=0.01)],\n#     'recurrent_regularizer': [None, l1(0.01), l2(0.01), l1_l2(l1=0.01, l2=0.01)],\n#     'bias_regularizer': [None, l1(0.01), l2(0.01), l1_l2(l1=0.01, l2=0.01)]\n}\n\n# Generate all combinations of hyperparameters\nhyperparams = np.array(np.meshgrid(*param_grid.values())).T.reshape(-1, len(param_grid))\n\nbest_accuracy = 0\nbest_hyperparams = None\n\nfor params in hyperparams:\n    lstm_units, dropout_rate,kernel_regularizer,recurrent_regularizer,bias_regularizer = params\n\n    x_input = Input(shape=(max_length,))\n    emb = Embedding(21, 128, input_length=max_length)(x_input)\n    bi_rnn = Bidirectional(CuDNNLSTM(int(lstm_units), kernel_regularizer=kernel_regularizer,\n                                     recurrent_regularizer=recurrent_regularizer, bias_regularizer=bias_regularizer))(emb)\n    x = Dropout(dropout_rate)(bi_rnn)\n    x_output = Dense(500, activation='softmax')(x)\n    model = Model(inputs=x_input, outputs=x_output)\n    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n    # Train the model\n    print(f\"\\n #### current model : {params} ### \")\n    model.fit(train_pad[IX_train,:], Y[IX_train,:], epochs=num_epochs, batch_size=batch_sz,validation_split=0.1, callbacks=[es])\n\n    # Evaluate on validation set\n    val_predictions = model.predict(train_pad[IX_test,:])\n    val_accuracy = accuracy_score(np.argmax(Y[IX_test,:], axis=1), np.argmax(val_predictions, axis=1))\n    f1_score = F1_score(normalize_vector(val_predictions),Y[IX_test,:])\n    # Check if current hyperparameters are the best so far\n    print(f\"model-{params}-val acc = {val_accuracy}\")\n    print(f\"model-{params}-f1 score = {f1_score}\")\n    if val_accuracy > best_accuracy:\n        best_accuracy = val_accuracy\n        best_hyperparams = params\n\n# Print the best hyperparameters and accuracy\nprint(\"Best Hyperparameters:\", best_hyperparams)\nprint(\"Best Accuracy:\", best_accuracy)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Keras Tuner for hyperparam tuning","metadata":{}},{"cell_type":"code","source":"def model_builder(hp):\n    x_input = Input(shape=(max_length,))\n    hp_emb_input_units = hp.Int('input_dim', min_value=100, max_value=1000, step=100)\n    hp_emb_output_units = hp.Int('output_dim', min_value=64, max_value=512, step=64)\n    emb = Embedding(hp_emb_input_units, hp_emb_output_units, input_length=max_length)(x_input)\n    hp_lstm_units = hp.Int('units', min_value=64, max_value=512, step=64)\n    hp_kernel_reg = hp.Choice('kernel_regularizer', values=[1e-2,1e-3,1e-4])\n    hp_recurrent_reg = hp.Choice('recurrent_regularizer', values=[1e-2,1e-3,1e-4])\n    hp_bias_reg = hp.Choice('bias_regularizer', values=[1e-2,1e-3,1e-4])\n    bi_rnn = Bidirectional(CuDNNLSTM(hp_lstm_units, kernel_regularizer=l2(hp_kernel_reg), recurrent_regularizer=l2(hp_recurrent_reg), bias_regularizer=l2(hp_bias_reg)))(emb)\n    x = Dropout(0.1)(bi_rnn)\n    hp_dense_units = hp.Int('units', min_value=64, max_value=512, step=64)\n    x = Dense(512,activation='relu')(x)\n\n    # softmax classifier\n    x_output = Dense(1499, activation='sigmoid')(x)\n\n    model = Model(inputs=x_input, outputs=x_output)\n\n  # Tune the learning rate for the optimizer\n  # Choose an optimal value from 0.01, 0.001, or 0.0001\n    hp_learning_rate = hp.Choice('learning_rate', values=[1e-4, 1e-5, 1e-6])\n    model.compile(optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate), loss='binary_crossentropy', metrics=['accuracy'])\n#     model.compile(optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate),\n#                 loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n#                 metrics=['accuracy'])\n\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuner = kt.Hyperband(model_builder,\n                     objective='val_accuracy',\n                     max_epochs=30,\n                     factor=3,\n                     directory='lstm',\n                     project_name='kt')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_early = EarlyStopping(monitor='val_loss', patience=5, verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuner.search(train_pad[IX_train,:], Y[IX_train,:], epochs=30, batch_size=128,validation_split=0.1,verbose=1, callbacks=[stop_early])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_hps=tuner.get_best_hyperparameters(num_trials=1)[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best Hyperparameters:\")\nfor param, value in best_hps.values.items():\n    print(f\"{param}: {value}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tuner.hypermodel.build(best_hps)\nhistory = model.fit(train_pad[IX_test,:], Y[IX_test,:], epochs=50, validation_split=0.1, callbacks = [stop_early])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'], label='Training Loss')\nplt.plot(history.history['val_loss'], label='Validation Loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()\n\n# Plot training and validation accuracy\nplt.plot(history.history['accuracy'], label='Training Accuracy')\nplt.plot(history.history['val_accuracy'], label='Validation Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(train_pad[IX_test,:])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"F1_score(normalize_vector(y_pred),Y[IX_test,:])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred_test = model.predict(test_pad)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(\"bilstm-full-v2.npy\",Y_pred_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"LSTM v1 =>   \n    Best Hyperparameters:  \n    units: 147  \n    kernel_regularizer: 0.01  \n    recurrent_regularizer: 0.01  \n    bias_regularizer: 0.01  \n    learning_rate: 1e-05  \n    tuner/epochs: 2  \n    tuner/initial_epoch: 0  \n    tuner/bracket: 2  \n    tuner/round: 0  \n      \nF-max = 0.16\n\nLSTM v2 =>  \n    Best Hyperparameters:  \n    input_dim: 210  \n    output_dim: 128  \n    units: 256  \n    kernel_regularizer: 0.01  \n    recurrent_regularizer: 0.001  \n    bias_regularizer: 0.001  \n    learning_rate: 1e-05  \n    tuner/epochs: 2  \n    tuner/initial_epoch: 0  \n    tuner/bracket: 2  \n    tuner/round: 0  \n  \nF-max = 0.16","metadata":{}}]}