{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Set the hyperparameters\nlayers = 3\nnodes = 512\nact_func = 'relu'\nbatch_norm = False\noptimizer = 'Adam'  # Non-functional\neta = 0.001\nl2 = None\ndropout = None\nbatch_size = 5120\nepochs = 20\nnum_of_predictions = None  # Set globally\nnum_of_go_terms = 1000","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:21:03.036964Z","iopub.execute_input":"2023-08-07T18:21:03.037316Z","iopub.status.idle":"2023-08-07T18:21:03.050919Z","shell.execute_reply.started":"2023-08-07T18:21:03.037288Z","shell.execute_reply":"2023-08-07T18:21:03.049917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load memory profiler\n%load_ext memory_profiler\n\n# Set up tensorflow\nimport os\nrandom_seed = 42\nos.environ['PYTHONHASHSEED'] = str(random_seed)\nos.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'\nos.environ['TF_GPU_ALLOCATOR'] = 'cuda_malloc_async'\n\n# Import libraries\nimport time\nimport random\nimport gc\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport progressbar\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Flatten, Dense, Activation, BatchNormalization, Dropout\nfrom keras import backend as K\n\nimport cupy as cp\nimport cudf\n\n# Set random seeds\nrandom.seed(random_seed)\nnp.random.seed(random_seed)\ntf.random.set_seed(random_seed)\ncp.random.seed(0)","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:21:03.053817Z","iopub.execute_input":"2023-08-07T18:21:03.0542Z","iopub.status.idle":"2023-08-07T18:21:13.76228Z","shell.execute_reply.started":"2023-08-07T18:21:03.054166Z","shell.execute_reply":"2023-08-07T18:21:13.76124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n### Prepare X_train ###\n\n# Load the embeddings\ntrain_embeddings = cp.load('/kaggle/input/t5embeds/train_embeds.npy')\n\n# Create the training features from the embeddings\nX_train = cudf.DataFrame.from_records(cp.asnumpy(train_embeddings))\n\n# Convert column names to strings\nX_train.columns = X_train.columns.astype(str)\n\n# Convert to float32\nX_train = X_train.astype('float32')\n\n# Save the training features\nX_train.to_parquet(f'/kaggle/working/X_train.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:21:13.763944Z","iopub.execute_input":"2023-08-07T18:21:13.764712Z","iopub.status.idle":"2023-08-07T18:21:51.948456Z","shell.execute_reply.started":"2023-08-07T18:21:13.764676Z","shell.execute_reply":"2023-08-07T18:21:51.946004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean up memory\ndel train_embeddings\ndel X_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:21:51.95311Z","iopub.execute_input":"2023-08-07T18:21:51.956216Z","iopub.status.idle":"2023-08-07T18:21:52.401712Z","shell.execute_reply.started":"2023-08-07T18:21:51.956175Z","shell.execute_reply":"2023-08-07T18:21:52.400606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n### Prepare y_train ###\n\n# Load the protein IDs\ntrain_protein_ids = np.load('/kaggle/input/t5embeds/train_ids.npy')\n\n# Read the GO terms\ntrain_terms = cudf.read_csv(\"/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\", sep=\"\\t\")\n\n# Get the first M labels\ngo_terms = train_terms['term'].value_counts().index[:num_of_go_terms].to_arrow().to_pylist()\n\n# Convert GO terms to array for performance\ngo_terms = np.array(go_terms)\n\n# Get train_terms data for the top M labels only\ntrain_terms_updated = train_terms.loc[train_terms['term'].isin(go_terms)]\n\n# Get the number of rows (N)\ntrain_size_N = train_protein_ids.shape[0]\ntrain_size_M = len(go_terms)\n\n# Create an empty matrix (N x M) for the labels\ny_train = np.zeros((train_size_N, train_size_M))\n\n# Convert from numpy to pandas series for better handling\ntrain_protein_ids = pd.Series(train_protein_ids)\n\n# Create the progress bar\nbar = progressbar.ProgressBar(\n    maxval=num_of_go_terms,\n    widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])\n\n    \n# Group train_terms_updated by 'term' and get the corresponding unique 'EntryID's for each label\ngo_terms_to_proteins_map = train_terms_updated.groupby('term')['EntryID'].unique()\n\n# Create a matrix of proteins and Go terms\nfor i, label in enumerate(go_terms):\n\n    # Get the proteins related to the current GO term\n    go_term_related_proteins = go_terms_to_proteins_map.loc[label] if label in go_terms_to_proteins_map else []\n\n    # Fill the corresponding column in the matrix\n    y_train[:, i] = train_protein_ids.isin(go_term_related_proteins)\n\n    # Increment the counter\n    i += 1\n\n    # Update the progress bar\n    bar.update(i)\n\n# End the progress bar \nbar.finish()\n\n# Convert labels into aa pandas dataframe\ny_train = pd.DataFrame(data=y_train, columns=go_terms)\n\n# Convert to float32\ny_train = y_train.astype('float32')\n\n# Save labels to disk\nnp.save('/kaggle/working/go_terms.npy', go_terms)\n\n# Save the training data to disk\ny_train.to_parquet(f'/kaggle/working/y_train.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:21:52.403594Z","iopub.execute_input":"2023-08-07T18:21:52.404023Z","iopub.status.idle":"2023-08-07T18:22:38.831549Z","shell.execute_reply.started":"2023-08-07T18:21:52.403985Z","shell.execute_reply":"2023-08-07T18:22:38.830528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean up memory\ndel train_terms\ndel train_protein_ids\ndel train_terms_updated\ndel go_terms\ndel go_terms_to_proteins_map\ndel go_term_related_proteins\ndel y_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:22:38.833414Z","iopub.execute_input":"2023-08-07T18:22:38.83495Z","iopub.status.idle":"2023-08-07T18:22:39.166214Z","shell.execute_reply.started":"2023-08-07T18:22:38.834893Z","shell.execute_reply":"2023-08-07T18:22:39.165267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n### Prepare X_test ###\n\n# Get the test embeddings\ntest_embeddings = cp.load('/kaggle/input/t5embeds/test_embeds.npy')\n\n# Convert test_embeddings to dataframe\nX_test = cudf.DataFrame(test_embeddings)\n\n# Convert column names to strings\nX_test.columns = X_test.columns.astype(str)\n\n# Convert to float32\nX_test = X_test.astype('float32')\n\n# Save the test features\nX_test.to_parquet(f'/kaggle/working/X_test.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:22:39.167487Z","iopub.execute_input":"2023-08-07T18:22:39.168181Z","iopub.status.idle":"2023-08-07T18:23:07.610003Z","shell.execute_reply.started":"2023-08-07T18:22:39.168146Z","shell.execute_reply":"2023-08-07T18:23:07.608855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean up memory\ndel test_embeddings\ndel X_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:23:07.613754Z","iopub.execute_input":"2023-08-07T18:23:07.614299Z","iopub.status.idle":"2023-08-07T18:23:08.127112Z","shell.execute_reply.started":"2023-08-07T18:23:07.614264Z","shell.execute_reply":"2023-08-07T18:23:08.125773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n### Create the model ###\n\n# Load features and labels from disk\nX_train = pd.read_parquet('/kaggle/working/X_train.parquet')\ny_train = pd.read_parquet('/kaggle/working/y_train.parquet')\n\n# Convert to float32\nX_train = X_train.astype('float32')\ny_train = y_train.astype('float32')\n\n# Get the input and output shapes\ninput_shape = [X_train.shape[1]]\noutput_shape = y_train.shape[1]\n\n# Create the model\nmodel = Sequential()\n\n# Add input layer\nmodel.add(BatchNormalization(input_shape=input_shape, name='input'))\nprint('Using batch normalization on input')\n\n# Add the hidden layers\nfor layer in range(layers):\n\n    # Add the hidden layer\n    if l2 is not None:\n        model.add(Dense(\n            units=nodes,\n            kernel_regularizer=tf.keras.regularizers.l2(l2)),\n            name=f'hidden_{layer}')\n        print('Using L2 regularization')\n    else:\n        model.add(Dense(units=nodes, name=f'hidden_{layer}'))\n        print('No L2 regularization')\n\n    # Add batch normalization\n    if batch_norm:\n        model.add(BatchNormalization())\n        print('Using batch norm on hidden layer')\n\n    # Add the activation function\n    model.add(Activation(act_func, name=f'activation_{layer}'))\n\n    # Add dropout\n    if dropout is not None:\n        model.add(Dropout(dropout))\n        print('Using dropout in hidden layer')\n\n# Add the output layer\nmodel.add(Dense(units=output_shape, activation='sigmoid', name='output'))\n\n# Compile the model\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(\n        learning_rate=eta),\n    loss='binary_crossentropy')\n\n# Summarize the model\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:23:08.132601Z","iopub.execute_input":"2023-08-07T18:23:08.132891Z","iopub.status.idle":"2023-08-07T18:23:12.743228Z","shell.execute_reply.started":"2023-08-07T18:23:08.132866Z","shell.execute_reply":"2023-08-07T18:23:12.742012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n### Train the model ###\n\n# Train the model\nhistory = model.fit(\n    X_train, y_train,\n    #validation_split=0.2,\n    batch_size=batch_size,\n    epochs=epochs,\n    verbose=2)\n\n# Save the model\nmodel.save(f'/kaggle/working/model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:23:12.745684Z","iopub.execute_input":"2023-08-07T18:23:12.746393Z","iopub.status.idle":"2023-08-07T18:23:39.247404Z","shell.execute_reply.started":"2023-08-07T18:23:12.746347Z","shell.execute_reply":"2023-08-07T18:23:39.245334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean up memory\ndel X_train\ndel y_train\ndel model\ndel history\ntf.keras.backend.clear_session()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:23:39.249469Z","iopub.execute_input":"2023-08-07T18:23:39.249971Z","iopub.status.idle":"2023-08-07T18:23:39.673967Z","shell.execute_reply.started":"2023-08-07T18:23:39.249904Z","shell.execute_reply":"2023-08-07T18:23:39.67299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n# Load the model\nmodel = tf.keras.models.load_model('/kaggle/working/model.h5')\n\n# Load the test features\nX_test = pd.read_parquet('/kaggle/working/X_test.parquet')\n\n# Convert to float32\nX_test = X_test.astype('float32')\n\n# Make the predictions\npredictions = model.predict(X_test, batch_size=1024)\n\n# Convert to cupy array\npredictions = cp.array(predictions)\n\n# Convert to float32\npredictions = predictions.astype('float16')\n\n# Save the predictions\ncp.save('/kaggle/working/predictions.npy', predictions)\n\n# Set the number of predictions (globally)\nnum_of_predictions = predictions.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:23:39.67554Z","iopub.execute_input":"2023-08-07T18:23:39.675919Z","iopub.status.idle":"2023-08-07T18:23:46.271Z","shell.execute_reply.started":"2023-08-07T18:23:39.675885Z","shell.execute_reply":"2023-08-07T18:23:46.269865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean up memory\ndel X_test\ndel model\ndel predictions\ntf.keras.backend.clear_session()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:23:46.274443Z","iopub.execute_input":"2023-08-07T18:23:46.274749Z","iopub.status.idle":"2023-08-07T18:23:46.636078Z","shell.execute_reply.started":"2023-08-07T18:23:46.27472Z","shell.execute_reply":"2023-08-07T18:23:46.634993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n# Create the submission table\nsubmission = cudf.DataFrame(columns = ['Protein Id', 'GO Term Id','Prediction'])","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:23:46.637749Z","iopub.execute_input":"2023-08-07T18:23:46.638076Z","iopub.status.idle":"2023-08-07T18:23:47.127723Z","shell.execute_reply.started":"2023-08-07T18:23:46.638049Z","shell.execute_reply":"2023-08-07T18:23:47.126027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n# Load the protein IDs\ntest_protein_ids = np.load('/kaggle/input/t5embeds/test_ids.npy')\n\n# Expand (broadcast) the list of protein IDs\nprotein_ids_list = []\nfor k in list(test_protein_ids):\n    protein_ids_list += [k] * num_of_go_terms\n\n# Clean up memory\ndel test_protein_ids\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:23:47.130034Z","iopub.execute_input":"2023-08-07T18:23:47.13049Z","iopub.status.idle":"2023-08-07T18:23:50.448402Z","shell.execute_reply.started":"2023-08-07T18:23:47.130447Z","shell.execute_reply":"2023-08-07T18:23:50.447235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n# Create protein ID column\nsubmission['Protein Id'] = protein_ids_list\n\n# Clean up memory\ndel protein_ids_list\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:23:50.450181Z","iopub.execute_input":"2023-08-07T18:23:50.450809Z","iopub.status.idle":"2023-08-07T18:24:01.742817Z","shell.execute_reply.started":"2023-08-07T18:23:50.450772Z","shell.execute_reply":"2023-08-07T18:24:01.740762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n# Load the GO terms\ngo_terms = np.load('/kaggle/working/go_terms.npy')\n\n# Convert GO terms to a list (for broadcasting)\ngo_terms = go_terms.tolist()\n\n# Create GO terms column\nsubmission['GO Term Id'] = go_terms * num_of_predictions\n\n# Clean up memory\ndel go_terms\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:24:01.74435Z","iopub.execute_input":"2023-08-07T18:24:01.744984Z","iopub.status.idle":"2023-08-07T18:24:14.055061Z","shell.execute_reply.started":"2023-08-07T18:24:01.744945Z","shell.execute_reply":"2023-08-07T18:24:14.05388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n# Load the predictions\npredictions = cp.load('/kaggle/working/predictions.npy')\n\n# Ravel the prediction\npredictions = predictions.ravel()\n\n# Create the predictions column\nsubmission['Prediction'] = predictions\n\n# Convert to a decimal with 3 decimal places\nsubmission['Prediction'] = submission['Prediction'].astype(cudf.Decimal32Dtype(4, 3))\n\n# Clean up memory\ndel predictions\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:24:14.056555Z","iopub.execute_input":"2023-08-07T18:24:14.057271Z","iopub.status.idle":"2023-08-07T18:24:15.684728Z","shell.execute_reply.started":"2023-08-07T18:24:14.057234Z","shell.execute_reply":"2023-08-07T18:24:15.683643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n# Save the submission file\nsubmission.to_csv(\n    f'submission.tsv',\n    sep='\\t',\n    index=False,\n    header=False,\n    chunksize=100000)\n\n# Clean up memory\ndel submission\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:24:15.686188Z","iopub.execute_input":"2023-08-07T18:24:15.686532Z","iopub.status.idle":"2023-08-07T18:24:28.702119Z","shell.execute_reply.started":"2023-08-07T18:24:15.686499Z","shell.execute_reply":"2023-08-07T18:24:28.700789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\n# Get a copy of the global variables dictionary\nglobal_vars = globals().copy()\n\n# Iterate over all variables in memory\nfor var_name, var_value in global_vars.items():\n    # Exclude special variables and modules\n    if not var_name.startswith('__') and not hasattr(var_value, '__call__'):\n        # Get the size of the variable\n        var_size = sys.getsizeof(var_value)\n        # Print the variable name and its size\n        print(f\"Variable: {var_name} | Size: {var_size} bytes\")","metadata":{"execution":{"iopub.status.busy":"2023-08-07T18:24:28.703777Z","iopub.execute_input":"2023-08-07T18:24:28.70481Z","iopub.status.idle":"2023-08-07T18:24:28.714734Z","shell.execute_reply.started":"2023-08-07T18:24:28.704768Z","shell.execute_reply":"2023-08-07T18:24:28.713609Z"},"trusted":true},"execution_count":null,"outputs":[]}]}