evaluate_splits

The train/test leakage evaluation. Needs MMseqs2 on PATH; see the installation notes.

evaluate_splits

Detect data leakage between the train and test halves of a dataset split.

Stages both halves to FASTA, runs an MMseqs2 easy-search of test against train, and reports how much of the test set has a near-identical counterpart in training - sequences a model can score correctly by memorisation alone.

run is the entry point; the CLI is a thin wrapper around it. The report layout is defined in genomic_benchmarks_qc.utils.naming.

run

run(
    train_files,
    test_files,
    format,
    out_folder='.',
    sequence_column=None,
    report_types=None,
    similarity_threshold=90.0,
    threads=None,
    split_memory_limit=None,
    keep_tmp_files=False,
    log_level='INFO',
    log_file=None,
)

Search the test half of a split against the train half and report the leakage.

Reports go to '/split//vs/', matching the layout of the classes command. FASTA inputs have no sequence column and use 'sequence'; several columns are searched concatenated and land in 'merged'. Any grouping above that - collection, dataset - is left to the caller, who expresses it through out_folder.

Parameters:

Name	Type	Description	Default
train_files	list[str]	Paths to the training files.	required
test_files	list[str]	Paths to the testing files.	required
format	str	Format of the input files (fasta, csv, csv.gz, tsv, tsv.gz).	required
out_folder	str | None	Path to the output folder; reports go into '/split/'. Default: '.'.	'.'
sequence_column	list[str] | None	Columns holding the sequences, for CSV/TSV input. Several columns are concatenated per row and searched together. Default: ['sequence'].	None
report_types	list[str] | None	Types of reports to generate, from REPORT_TYPES. Default: ['html', 'simple'].	None
similarity_threshold	float | None	Percent similarity at which a test sequence counts as leaked. Similarity is min(query coverage, target coverage) x percent identity, so a short exact match inside a long sequence does not reach it. Between 0 and 100. Default: 90.0.	90.0
threads	int | None	Maximum number of threads MMseqs2 will use. Default: None.	None
split_memory_limit	str | None	Upper RAM limit for MMseqs2 prefilter structures (e.g., 10G, 1T). Default: None.	None
keep_tmp_files	bool | None	Keep the MMseqs2 scratch files instead of deleting them. Default: False.	False
log_level	str | None	Logging level. Default: 'INFO'.	'INFO'
log_file	str | None	Path to a log file. Logs go to the console either way.	None

Source code in src/genomic_benchmarks_qc/evaluate_splits.py

def run(
    train_files: list[str],
    test_files: list[str],
    format: str,
    out_folder: str | None = '.',
    sequence_column: list[str] | None = None,
    report_types: list[str] | None = None,
    similarity_threshold: float | None = 90.0,
    threads: int | None = None,
    split_memory_limit: str | None = None,
    keep_tmp_files: bool | None = False,
    log_level: str | None = 'INFO',
    log_file: str | None = None,
):
    """Search the test half of a split against the train half and report the leakage.

    Reports go to '<out_folder>/split/<column>/<train>_vs_<test>/', matching the
    layout of the classes command. FASTA inputs have no sequence column and use
    'sequence'; several columns are searched concatenated and land in 'merged'.
    Any grouping above that - collection, dataset - is left to the caller, who
    expresses it through `out_folder`.

    Args:
        train_files: Paths to the training files.
        test_files: Paths to the testing files.
        format: Format of the input files (fasta, csv, csv.gz, tsv, tsv.gz).
        out_folder: Path to the output folder; reports go into '<out_folder>/split/'.
            Default: `'.'`.
        sequence_column: Columns holding the sequences, for CSV/TSV input. Several
            columns are concatenated per row and searched together.
            Default: `['sequence']`.
        report_types: Types of reports to generate, from
            [REPORT_TYPES][genomic_benchmarks_qc.evaluate_splits.REPORT_TYPES].
            Default: `['html', 'simple']`.
        similarity_threshold: Percent similarity at which a test sequence counts as
            leaked. Similarity is `min(query coverage, target coverage) x percent
            identity`, so a short exact match inside a long sequence does not reach it.
            Between 0 and 100. Default: `90.0`.
        threads: Maximum number of threads MMseqs2 will use. Default: `None`.
        split_memory_limit: Upper RAM limit for MMseqs2 prefilter structures (e.g., 10G,
            1T). Default: `None`.
        keep_tmp_files: Keep the MMseqs2 scratch files instead of deleting them.
            Default: `False`.
        log_level: Logging level. Default: `'INFO'`.
        log_file: Path to a log file. Logs go to the console either way.
    """

    if sequence_column is None:
        sequence_column = ['sequence']
    if report_types is None:
        report_types = ['html', 'simple']
    validate_report_types(report_types, REPORT_TYPES, 'evaluate-splits')

    setup_logger(log_level, log_file)
    logger.info("Starting train-test split evaluation.")

    # One directory per train-vs-test comparison, so different comparisons of the
    # same dataset can run concurrently into one output folder without
    # overwriting each other. The scratch directory below lives inside it and is
    # unique per run, so even two runs of the *same* comparison keep their
    # working files apart - though they would still write the same reports.
    comparison_dir = ensure_directory(
        Path(out_folder)
        / SPLIT_SUBDIR
        / column_dirname(format, sequence_column)
        / _build_comparison_dirname(train_files, test_files)
    )

    # `mkdtemp` rather than a fixed name: it creates the directory atomically, so
    # this run is provably its only owner and the cleanup below cannot delete
    # files that belong to anything else. Re-using a fixed name would mean
    # adopting whatever was already there and then removing it - which would
    # take out the scratch files of a concurrent run of the same comparison, or
    # the files a previous `--keep-tmp-files` run was asked to preserve.
    tmp_dir = Path(tempfile.mkdtemp(dir=comparison_dir, prefix=TMP_PREFIX))

    # `mkdtemp` hardcodes mode 0700, which would make kept scratch files
    # unreadable to everyone but the owner even where the reports beside them
    # are group-readable. Match the comparison directory instead, so a shared
    # output tree stays shared.
    try:
        os.chmod(tmp_dir, stat.S_IMODE(os.stat(comparison_dir).st_mode))
    except OSError as mode_error:
        logger.debug(f"Could not match permissions of the temporary directory: {mode_error}")

    train_fasta_path = tmp_dir / 'train_sequences.fasta'
    test_fasta_path = tmp_dir / 'test_sequences.fasta'

    try:
        with log_failures("Train-test split evaluation"):
            train_stats = _stage_sequences_to_fasta(
                train_fasta_path, train_files, format, sequence_column, 'train')
            test_stats = _stage_sequences_to_fasta(
                test_fasta_path, test_files, format, sequence_column, 'test')
            num_train_seqs = train_stats["count"]
            num_test_seqs = test_stats["count"]

            if num_train_seqs == 0 or num_test_seqs == 0:
                raise ValueError(
                    "Both train and test inputs must contain at least one sequence "
                    "before running split evaluation."
                )

            logger.info(f"Read {num_train_seqs} sequences from training files.")
            logger.info(f"Read {num_test_seqs} sequences from testing files.")

            # Run MMseqs2 search and keep raw TSV path for chunked post-processing.
            outfile = "mmseqs2_search_result.tsv"
            results_path = mmseqs_runtime.run_search(
                test_fasta_path,
                train_fasta_path,
                tmp_dir / outfile,
                tmp_dir,
                threads=threads,
                split_memory_limit=split_memory_limit,
            )

            # The exported table of the leaked hits is written while the search
            # output is read, because it is the one output with a row per hit and
            # no cap on how many. Its directory therefore has to exist before the
            # summary runs, rather than being made with the rest of the bundle.
            export_path = None
            if 'html' in report_types:
                mmseqs_dir = comparison_dir / MMSEQS_DIR
                mmseqs_dir.mkdir(parents=True, exist_ok=True)
                export_path = mmseqs_dir / outfile

            # `top_n` is the report's row cap, not a second number that has to be
            # kept in step with it: the top hits exist to be listed on the page.
            summary = summarize_mmseqs_output(
                results_path,
                similarity_threshold,
                query_count=num_test_seqs,
                target_count=num_train_seqs,
                top_n=ROW_CAP,
                export_path=export_path,
            )

            # Said here rather than inside the summariser, which reads the table
            # in chunks and would otherwise say it once per chunk.
            log_reversed_hit_warning(
                summary["reversed_hits"],
                summary["reversed_leaked_hits"],
                summary["total_hits"],
                threads=threads,
            )

            # Get threshold stats
            threshold_stats = get_threshold_stats(
                summary = summary,
                similarity_threshold = similarity_threshold,
                num_train_seqs = num_train_seqs,
                num_test_seqs = num_test_seqs
            )

            if 'simple' in report_types:
                df = _build_simple_report_frame(threshold_stats)
                generate_simple_report(df, comparison_dir / SIMPLE_REPORT_FILE)

            if 'html' in report_types:
                _write_mmseqs_report_bundle(
                    comparison_dir,
                    train_files,
                    test_files,
                    train_stats,
                    test_stats,
                    threshold_stats,
                    summary,
                    train_fasta_path,
                    test_fasta_path,
                )

        logger.info("Train-test split evaluation successfully completed.")
    finally:
        if keep_tmp_files:
            logger.info(f"Keeping temporary files for debugging at: {tmp_dir}")
        else:
            logger.debug("Removing temporary files.")
            try:
                shutil.rmtree(tmp_dir)
            except Exception as cleanup_error:
                logger.warning(f"Failed to remove temporary directory: {cleanup_error}")

REPORT_TYPES module-attribute

REPORT_TYPES = ('html', 'simple')