{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":118765,"databundleVersionId":15231210,"sourceType":"competition"},{"sourceId":10855324,"sourceType":"datasetVersion","datasetId":6742586},{"sourceId":11775065,"sourceType":"datasetVersion","datasetId":7392749},{"sourceId":11788894,"sourceType":"datasetVersion","datasetId":7402173},{"sourceId":290004465,"sourceType":"kernelVersion"}],"dockerImageVersionId":31012,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Identify 3D templates for RNA targets\nThis notebook prepares a mock submission as an illustration of how to derive 3D templates for targets in the Stanford 3D RNA folding competition. See also https://www.kaggle.com/datasets/rhijudas/rna-3d-folding-templates/data for precomputed outputs useful for training models for RNA 3D folding.\n\n## Important options","metadata":{}},{"cell_type":"code","source":"# if False, don't check on temporal_cutoff -- during training on known structures, will get lots of leakage. \n# If going for Early Sharing Prize, set to True.\nCHECK_TEMPORAL_CUTOFF = True\n\n# Number of templates to use. \n# Here set to 5 to prepare a mock submission\n# Should make larger (e.g., 40) if using templates for modeling. \nMAX_TEMPLATES = 5\n\n# Better to use nan when preparing files for templates, to allow easy recognition of which coordinates are missing.\n# But for this example, using 0.0 to avoid errors in scoring the final submission.csv\nNULL_VALUE = 0.0  \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:06.29255Z","iopub.execute_input":"2026-01-10T22:53:06.29283Z","iopub.status.idle":"2026-01-10T22:53:06.302564Z","shell.execute_reply.started":"2026-01-10T22:53:06.292808Z","shell.execute_reply":"2026-01-10T22:53:06.300726Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Download MMseqs2","metadata":{}},{"cell_type":"code","source":"#!wget https://mmseqs.com/latest/mmseqs-linux-avx2.tar.gz\n#!tar xvfz /kaggle/working/mmseqs-linux-avx2.tar.gz\n!rsync -avL /kaggle/input/mmseqs2/mmseqs /kaggle/working/\n!chmod 755 /kaggle/working/mmseqs/bin/mmseqs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:06.303783Z","iopub.execute_input":"2026-01-10T22:53:06.304102Z","iopub.status.idle":"2026-01-10T22:53:07.246353Z","shell.execute_reply.started":"2026-01-10T22:53:06.304073Z","shell.execute_reply":"2026-01-10T22:53:07.244736Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create DB based on FASTA of all PDB nucleic acid sequences, which is part of PDB_RNA dataset","metadata":{}},{"cell_type":"code","source":"!/kaggle/working/mmseqs/bin/mmseqs createdb /kaggle/input/stanford-rna-3d-folding-2/PDB_RNA/pdb_seqres_NA.fasta pdb_seqres_NA --dbtype 2 # > MMseqs_createDB.log","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:07.249596Z","iopub.execute_input":"2026-01-10T22:53:07.250039Z","iopub.status.idle":"2026-01-10T22:53:07.664438Z","shell.execute_reply.started":"2026-01-10T22:53:07.249995Z","shell.execute_reply":"2026-01-10T22:53:07.663177Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Find templates for targets in test_sequences.csv by aligning to PDB with MMseqs2","metadata":{}},{"cell_type":"markdown","source":"### Need to  convert test_sequences.csv file to FASTA","metadata":{}},{"cell_type":"code","source":"import csv\ninput_file='/kaggle/input/stanford-rna-3d-folding-2/test_sequences.csv'\noutput_file='test_sequences.fasta'\nwith open(input_file, 'r', newline='') as csv_file, open(output_file, 'w') as fasta_file:\n    csv_reader = csv.reader(csv_file, quotechar='\"', delimiter=',', quoting=csv.QUOTE_ALL, skipinitialspace=True)\n    next(csv_reader)  # Skip the header row\n    for row in csv_reader:\n        if len(row) >= 2:\n            fasta_file.write(f\">{row[0]}\\n{row[1]}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:07.668361Z","iopub.execute_input":"2026-01-10T22:53:07.668673Z","iopub.status.idle":"2026-01-10T22:53:07.679992Z","shell.execute_reply.started":"2026-01-10T22:53:07.668637Z","shell.execute_reply":"2026-01-10T22:53:07.679052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!/kaggle/working/mmseqs/bin/mmseqs easy-search /kaggle/working/test_sequences.fasta /kaggle/working/pdb_seqres_NA testResult.txt tmp --search-type 3 --format-output \"query,target,evalue,qstart,qend,tstart,tend,qaln,taln\" ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:07.681767Z","iopub.execute_input":"2026-01-10T22:53:07.682198Z","iopub.status.idle":"2026-01-10T22:53:25.66426Z","shell.execute_reply.started":"2026-01-10T22:53:07.682165Z","shell.execute_reply":"2026-01-10T22:53:25.663063Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Assemble file of template coordinates by going through .cif files found for each target by MMseqs2","metadata":{}},{"cell_type":"code","source":"!pip3 install /kaggle/input/biopython/biopython-1.85-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl --no-deps","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:25.665625Z","iopub.execute_input":"2026-01-10T22:53:25.665899Z","iopub.status.idle":"2026-01-10T22:53:30.298904Z","shell.execute_reply.started":"2026-01-10T22:53:25.66587Z","shell.execute_reply":"2026-01-10T22:53:30.297599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from Bio import SeqIO,PDB,BiopythonWarning\nfrom Bio.PDB.MMCIF2Dict import MMCIF2Dict\nfrom Bio.Seq import Seq\nfrom Bio.PDB import MMCIFParser\nimport numpy as np\nimport pandas as pd\nimport os\nimport gzip\nimport sys\nimport warnings\nimport argparse\nfrom datetime import datetime\n\n# Suppress warnings\nwarnings.simplefilter('ignore', BiopythonWarning)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:30.300234Z","iopub.execute_input":"2026-01-10T22:53:30.300557Z","iopub.status.idle":"2026-01-10T22:53:30.756875Z","shell.execute_reply.started":"2026-01-10T22:53:30.300527Z","shell.execute_reply":"2026-01-10T22:53:30.755821Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Basic options and file locations","metadata":{}},{"cell_type":"code","source":"sequences_file = '/kaggle/input/stanford-rna-3d-folding-2/test_sequences.csv'\nmmseqs_results_file = '/kaggle/working/testResult.txt' # created by MMseqs command lines\noutfile = 'submission.csv'\ncif_dir = '/kaggle/input/stanford-rna-3d-folding-2/PDB_RNA'\n\n# variables not in use here:\nid_map_file = ''\nstart_idx = 0\nend_idx = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:30.76025Z","iopub.execute_input":"2026-01-10T22:53:30.761322Z","iopub.status.idle":"2026-01-10T22:53:30.766477Z","shell.execute_reply.started":"2026-01-10T22:53:30.761285Z","shell.execute_reply":"2026-01-10T22:53:30.765492Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Helper functions","metadata":{}},{"cell_type":"code","source":"def clean_res_name( res_name ):\n    if res_name in ['A', 'C', 'G', 'U']:\n        return res_name\n    else: # can be modified residue with 3-letter name.\n        return 'X'\n\ndef extract_title_release_date( cif_path ):\n\n    if cif_path.endswith('.gz'):\n        with gzip.open(cif_path, 'rt') as cif_file:\n            mmcif_dict = MMCIF2Dict(cif_file)\n    else:\n        mmcif_dict = MMCIF2Dict(cif_path)\n\n    possible_title_fields = [\n        '_struct.title',\n        '_entry.title',\n        '_struct_keywords.pdbx_keywords'\n    ]\n\n    pdb_title = None\n    for field in possible_title_fields:\n        if field in mmcif_dict:\n            pdb_title = mmcif_dict[field]\n            if isinstance(pdb_title, list):\n                pdb_title = ' '.join(pdb_title)\n            break\n\n    possible_date_fields = [\n        '_pdbx_database_status.initial_release_date',\n        '_pdbx_database_status.recvd_initial_deposition_date',\n        '_database_PDB_rev.date'\n    ]\n\n    release_date = None\n    for field in possible_date_fields:\n        if field in mmcif_dict:\n            release_date = mmcif_dict[field]\n            if isinstance(release_date, list):\n                release_date = release_date[0]  # Take the first date if it's a list\n            break\n\n    return pdb_title, release_date\n\n\ndef extract_rna_sequence(cif_path,chain_id):\n\n    if cif_path.endswith('.gz'):\n        with gzip.open(cif_path, 'rt') as cif_file:\n            mmcif_dict = MMCIF2Dict(cif_file)\n    else:\n        mmcif_dict = MMCIF2Dict(cif_path)\n\n    pdb_sequence = None\n    pdb_chain_id = None\n    chain_seq_nums = None\n\n    # Extract _pdbx_poly_seq_scheme information\n    strand_id  = mmcif_dict.get('_pdbx_poly_seq_scheme.pdb_strand_id',[])\n    mon_id     = mmcif_dict.get('_pdbx_poly_seq_scheme.mon_id',[])\n    pdb_mon_id = mmcif_dict.get('_pdbx_poly_seq_scheme.pdb_mon_id',[])\n    pdb_seq_num = mmcif_dict.get('_pdbx_poly_seq_scheme.pdb_seq_num',[])\n    chain_ids = list(set(strand_id))\n    seq_chains = []\n\n    full_sequence = ''\n    pdb_chain_sequence = ''\n    pdb_chain_seq_nums = []\n    for (strand,mon,pdb_mon,pdb_num) in zip(strand_id,mon_id,pdb_mon_id,pdb_seq_num):\n        if strand==chain_id:\n            full_sequence += clean_res_name( mon )\n            pdb_chain_sequence += clean_res_name( pdb_mon )\n            pdb_chain_seq_nums.append( pdb_num)\n\n    #print(full_sequence)\n    #print(pdb_chain_sequence)\n    #print(pdb_chain_seq_nums)\n\n    return full_sequence,pdb_chain_sequence,pdb_chain_seq_nums\n\ndef get_c1prime_labels(cif_path, chain_id, alignment, chain_seq_nums):\n    \"\"\"\n    Extract C1' coordinates for an RNA chain based on a reference sequence alignment.\n\n    This function uses Biopython to parse a CIF file, finds the specified chain,\n    and extracts C1' coordinates for RNA residues. It aligns these coordinates\n    with a reference sequence, handling gaps and missing residues.\n\n    Parameters:\n    cif_path (str): Path to the CIF file.\n    chain_id (str): Chain identifier in the CIF file.\n    alignment (list): A list containing two elements:\n                      alignment[0]: List of residues for the reference sequence (A,C,G,U,-)\n                      alignment[1]: List of residues for the chain sequence (A,C,G,U,X,-)\n    chain_seq_nums (list): numbers of residues in PDB\n\n    Returns:\n    list of tuples: Each tuple contains (resname, resid, x, y, z), where:\n                    resname: Residue name (A, C, G, or U) from the reference sequence\n                    resid: Residue ID (1, 2, 3, ...) based on position in reference sequence\n                    x, y, z: C1' coordinates (nan or NULL_VALUE for missing residues/atoms)\n\n    The length of the returned list is equal to the number of non-gap residues\n    in the reference sequence.\n    \"\"\"\n    # Parse the CIF file\n    parser = MMCIFParser()\n    if cif_path.endswith('.gz'):\n        with gzip.open(cif_path, 'rt') as gz_file:\n            structure = parser.get_structure('RNA', gz_file )\n    else:\n        structure = parser.get_structure('RNA', cif_path)\n\n    # Get the specified chain\n    chain = structure[0][chain_id]\n\n    # getting residues out of chain is complex -- easier to get a list ahead of time.\n    residues = {}\n    for residue in chain: residues[ residue.id[1] ] = residue\n\n    chain_seq = ''.join( [clean_res_name(residue.get_resname()) for residue in chain ] )\n    #print(chain_seq)\n\n    # Initialize the result list\n    result = []\n\n    # Counter for residue ID in reference sequence\n    ref_resid = 0\n    chain_idx = 0\n    for ref_res, chain_res in zip(alignment[0], alignment[1]):\n        if chain_res != '-': chain_idx += 1\n        if ref_res != '-':\n            ref_resid += 1\n            if chain_res == '-': # or chain_res == 'X':\n                # Missing residue in chain or unknown residue\n                result.append((ref_res, ref_resid, NULL_VALUE, NULL_VALUE, NULL_VALUE, -1e18))\n            else:\n                # Find the corresponding residue in the chain\n                try:\n                    #chain_seq_num = int(chain_seq_nums[ref_resid-1])\n                    chain_seq_num = int(chain_seq_nums[chain_idx-1])\n                    residue = residues[chain_seq_num]\n                    c1_prime = residue['C1\\'']\n                    coords = c1_prime.coord\n                    if residue.get_resname() != chain_res:\n                        print( 'WARNING!',ref_resid,chain_idx,chain_seq_num,residue.get_resname(),chain_res)\n                    result.append((ref_res, ref_resid, coords[0], coords[1], coords[2], residue.id[1]))\n                except KeyError:\n                    # C1' atom not found\n                    result.append((ref_res, ref_resid, NULL_VALUE, NULL_VALUE, NULL_VALUE, -1e18))\n                except Exception as e:\n                    # Any other error (e.g., residue not found)\n                    result.append((ref_res, ref_resid, NULL_VALUE, NULL_VALUE, NULL_VALUE, -1e18))\n\n    return result\n\ndef is_before_or_on(d1, d2):\n    date1 = pd.to_datetime(d1)\n    date2 = pd.to_datetime(d2)\n    return date1 <= date2\n\ndef read_id_map(id_map_file):\n    if len(id_map_file)==0: return None\n    id_map = {}\n    try:\n        with open(id_map_file, newline='') as f:\n            reader = csv.DictReader(f)\n            if 'orig' not in reader.fieldnames or 'new' not in reader.fieldnames:\n                print(\"Warning: ID map file does not contain the fields 'orig' and 'new'. Using original IDs instead.\")\n                return id_map\n            for row in reader:\n                id_map[row['orig']] = row['new']\n    except FileNotFoundError:\n        print(f\"Warning: ID map file {id_map_file} not found. Using original IDs instead.\", file=sys.stderr)\n    except Exception as exc:\n        print(f\"Error reading {id_map_file}: {exc}\", file=sys.stderr)\n    return id_map\n\ndef read_release_dates( release_data_file ):\n    release_dates = {}\n    # must have format Entry ID, Release Date\n    with open(release_data_file, newline='') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            release_dates[row['Entry ID']] = row['Release Date']\n\n    return release_dates","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:30.767341Z","iopub.execute_input":"2026-01-10T22:53:30.767616Z","iopub.status.idle":"2026-01-10T22:53:30.870123Z","shell.execute_reply.started":"2026-01-10T22:53:30.767596Z","shell.execute_reply":"2026-01-10T22:53:30.869006Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Setup","metadata":{}},{"cell_type":"code","source":"# Prepare to collect output data\noutput_labels = []\n\n# Read the FASTA file\ndf = pd.read_csv( sequences_file )\ntargets = df['target_id'].to_list()\nsequences = df['sequence'].to_list()\ntemporal_cutoffs = df['temporal_cutoff'].to_list()\n\naln_lines = []\nfor line in open( mmseqs_results_file ).readlines():\n    # query,template,eval,qstart,qend,tstart,tend,qaln,taln\n    aln_lines.append( line.strip().split() )\n\nid_map = read_id_map( id_map_file )\n\nrelease_dates = read_release_dates( cif_dir + '/pdb_release_dates_NA.csv' )\n\nif start_idx == 0 and end_idx == 0: # do all targets by default\n    start_idx = 1\n    end_idx = len(targets)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:30.871476Z","iopub.execute_input":"2026-01-10T22:53:30.871798Z","iopub.status.idle":"2026-01-10T22:53:30.938562Z","shell.execute_reply.started":"2026-01-10T22:53:30.871774Z","shell.execute_reply":"2026-01-10T22:53:30.937478Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_targets = 0\ncount = 0\nfor target,sequence,temporal_cutoff in zip(targets,sequences,temporal_cutoffs):\n    count += 1\n    if (count < start_idx) or (count > end_idx): continue\n\n    # look for alignments and fill out C1' templates\n    templates = []\n    for aln_line in aln_lines:\n        if len(aln_line)!=9: continue # some kind of overflow in some alignments?\n\n        query,template,eval,qstart,qend,tstart,tend,qaln,taln = aln_line\n\n        if query != target: continue\n\n        if int(qend)<int(qstart): continue # aligned to reverse complement!\n\n        pdb_id,chain_id = template.split('_')\n\n        # need to do alignment\n        cif_path = os.path.join(cif_dir, f'{pdb_id.lower()}.cif')\n        if not os.path.isfile( cif_path ): continue # occasional alignment to DNA, ignore!\n\n        release_date = release_dates[pdb_id.upper()] # pulled from PDB server\n\n        if CHECK_TEMPORAL_CUTOFF and is_before_or_on(temporal_cutoff,release_date): continue\n\n        # these release dates in the CIF files can be buggy!\n        title,release_date_unreliable = extract_title_release_date( cif_path )\n\n        print('\\n',target,temporal_cutoff,\"   \",template)\n        if title: print(f\"PDB Title: {title}\")\n        if release_date: print(f\"PDB Release Date: {release_date}\")\n\n        # sometimes there is a mismatch between PDB's fasta files and what's actually stored in coordinates,\n        # so best to get the actual residue numbers for the chain\n        chain_full_sequence,chain_sequence,chain_seq_nums = extract_rna_sequence(cif_path,chain_id)\n\n        # get 3d data\n        alignment = []\n        qstart=int(qstart)\n        qend=int(qend)\n        tstart=int(tstart)\n        tend=int(tend)\n        alignment.append( sequence[:(qstart-1)] + '-'*(tstart-1) + qaln + sequence[qend:]  )\n        alignment.append( '-'*(qstart-1)        + 'X'*(tstart-1) + taln + '-'*(len(sequence)-qend) )\n        print( alignment[0] )\n        print( alignment[1] )\n        c1prime_data = get_c1prime_labels( cif_path, chain_id, alignment, chain_seq_nums )\n\n        # mismatch in FASTA sequence and the polyx info in the CIF file\n        if len(c1prime_data) != len(sequence):\n            print( 'WARNING! len(c1prime_data) != len(sequence)', 'len c1prime_data', len(c1prime_data), 'len sequence', len(sequence), 'qstart',qstart,'len qaln',len(qaln),'qend',qend)\n            continue\n\n        templates.append( c1prime_data )\n\n        if len(templates) >= MAX_TEMPLATES: break\n\n    print( \"Found\", len(templates), \"templates for\", target,'\\n' )\n\n    mapped_target = target\n    if not id_map is None: mapped_target = id_map[target]\n\n    for i in range(len(sequence)):\n        output_label = {\n            \"ID\": f'{mapped_target}_{i+1}',\n            \"resname\": sequence[i],\n            \"resid\": i+1,\n        }\n\n        # output templates\n        for n in range(len(templates)):\n            res,resid,x,y,z,pdb_seqnum = templates[n][i]\n            assert( resid == i+1 )\n            output_label[ f\"x_{n+1}\" ] = x\n            output_label[ f\"y_{n+1}\" ] = y\n            output_label[ f\"z_{n+1}\" ] = z\n\n        # pad with blank models\n        for n in range(len(templates),MAX_TEMPLATES):\n            output_label[ f\"x_{n+1}\" ] = NULL_VALUE\n            output_label[ f\"y_{n+1}\" ] = NULL_VALUE\n            output_label[ f\"z_{n+1}\" ] = NULL_VALUE\n        output_labels.append( output_label )\n\n    num_targets += 1\n    # if num_targets > 1: break # for debug!\n\n\nprint(f'Completed {num_targets} targets\\n')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:53:30.939518Z","iopub.execute_input":"2026-01-10T22:53:30.939816Z","iopub.status.idle":"2026-01-10T22:57:27.191774Z","shell.execute_reply.started":"2026-01-10T22:53:30.939794Z","shell.execute_reply":"2026-01-10T22:57:27.190837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a DataFrame and write to CSV\ndef output_csv( output_data, outfile ):\n    df = pd.DataFrame(output_data)\n    df.to_csv(outfile, index=False)\n    print(f\"Output written to {outfile}\")\n\noutput_csv( output_labels, outfile )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:57:27.192734Z","iopub.execute_input":"2026-01-10T22:57:27.193074Z","iopub.status.idle":"2026-01-10T22:57:27.355464Z","shell.execute_reply.started":"2026-01-10T22:57:27.19305Z","shell.execute_reply":"2026-01-10T22:57:27.354252Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Let's run evaluation with the competition's metric to get scores ","metadata":{}},{"cell_type":"markdown","source":"This code block reads in the metric **TM-Score PermuteChains**, which can be installed as input data for the notebook from the link https://www.kaggle.com/code/rhijudas/tm-score-permutechains/data. \n\nWe also need the USalign code installed as input data, available at: https://www.kaggle.com/datasets/metric/usalign","metadata":{}},{"cell_type":"code","source":"!ls /kaggle/usr/lib/\nimport runpy\nmodule_globals = runpy.run_path(\"/kaggle/usr/lib/tm-score-permutechains/metric.py\")\nscore = module_globals['score']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T22:57:27.360923Z","iopub.execute_input":"2026-01-10T22:57:27.361186Z","iopub.status.idle":"2026-01-10T22:57:27.508162Z","shell.execute_reply.started":"2026-01-10T22:57:27.361167Z","shell.execute_reply":"2026-01-10T22:57:27.506961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nsol = pd.read_csv('/kaggle/input/stanford-rna-3d-folding-2/validation_labels.csv')\nsub = pd.read_csv(outfile)\nprint('Length of solution:',len(sol),'Length of submission',len(sub))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T23:33:20.767075Z","iopub.execute_input":"2026-01-10T23:33:20.767422Z","iopub.status.idle":"2026-01-10T23:33:20.978133Z","shell.execute_reply.started":"2026-01-10T23:33:20.767374Z","shell.execute_reply":"2026-01-10T23:33:20.976899Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The score() function will go over entire dataframe -- all the targets. But let's get scores target by target so we can get feedback on which ones are 'solved' vs. not. ","metadata":{}},{"cell_type":"code","source":"sol['target_id'] = sol['ID'].apply(lambda x: '_'.join(str(x).split('_')[:-1]))\nsub['target_id'] = sub['ID'].apply(lambda x: '_'.join(str(x).split('_')[:-1]))\n\nif len(sol)==len(sub): # This tests if we're looking at public val\n    results = []\n    for target_id, group_native in sol.groupby('target_id'):\n        # if len(group_native)>200: continue # quick test\n        group_predicted = sub[sub['target_id'] == target_id]\n        result = score(group_native,group_predicted,'ID')\n        print(target_id,result)\n        results.append( result )\n    print( 'Mean score:',  \n          float(sum(results) / len(results)) if len(results)>0 else 0.0, \n          f'(n={len(results)})' )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T23:40:49.29459Z","iopub.execute_input":"2026-01-10T23:40:49.294942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}