slug = "human-colon"
release = "2026-10-28"
use_gpu = True
out_dir = "bundles"
smoke = 0
primers_ref = "feat-primers"
cohorts_path = None
catalog_path = None
representatives = 1
dev_note = ""

# Parameters
slug = "human-colon"
release = "2026-11-01"
out_dir = "bundles"
primers_ref = "4d7af340849c00077f9d21c5b54af9e689a4a070"
dev_note = ""

pip install "sddb-primers @ git+https://github.com/theislab/spatialdata-db-client@4d7af340849c00077f9d21c5b54af9e689a4a070#subdirectory=primers"

import inspect
import json
import platform
import time

import anndata as ad
import numpy as np
import pandas as pd
import scanpy as sc
import spatialdata_plot  # noqa: F401  (registers the .pl accessor)
import zarr
from primers.cohort import hd_table, load_cohort, smoke_spec
from primers.embed import CAVEAT, MAX_FLOOR_LOSS, MIN_SHARED_GENES, N_HVG, N_NEIGHBORS, N_PCS, SEED, embed_views
from primers.export import export_bundle, mixing_by_view, with_qc_status
from primers.markers import load_markers, panel_id
from primers.qc import MIN_GENES_FOR_QC, apply_qc, load_thresholds, qc_floors
from spatialdata.transformations import get_transformation

import sddb
from sddb.remote import storage_options

start = time.time()
out_dir = Path(out_dir)
if use_gpu:
    import cupy as cp
    import rapids_singlecell as rsc
    import rmm
    from rmm.allocators.cupy import rmm_cupy_allocator

    rmm.reinitialize(managed_memory=True)
    cp.cuda.set_allocator(rmm_cupy_allocator)


def show_call(func, what):
    """Show what a primers function does and link its source at ``primers_ref``."""
    _, line = inspect.getsourcelines(func)
    module = func.__module__.split(".")[-1]
    url = f"{REPO}/blob/{primers_ref}/primers/primers/{module}.py#L{line}"
    display(Markdown(f"`{func.__module__}.{func.__name__}`: {what} [Source at `{primers_ref}`]({url})."))


def show_table(df, digits=3):
    """Display ``df`` rounded, with missing values written as n/a, and long text columns in full."""
    shown = df.round(digits).astype(object)
    with pd.option_context("display.max_colwidth", None):
        display(shown.where(df.notna(), "n/a"))


catalog = sddb.Catalog.from_file(catalog_path) if catalog_path else sddb.Catalog(version=release)

if smoke:
    show_call(
        smoke_spec,
        "Keeps only the smallest kept member of each of a few papers, taken in turn from each family, for a "
        "quick end-to-end check.",
    )
    n_spec_excluded = len(spec.get("exclude_uids") or [])
    spec = smoke_spec(spec, catalog, smoke)
    print(
        f"Smoke run on {smoke} papers: {len(spec['exclude_uids']) - n_spec_excluded} more uids are excluded "
        f"({len(spec['exclude_uids'])} in all)."
    )
for key in ("slug", "query", "include_uids", "exclude_uids", "families", "marker_set", "rationale"):
    value = spec.get(key)
    if smoke and key == "exclude_uids":
        value = f"{len(value)} uids"
    print(f"{key}: {value}")

slug: human-colon
query: {'organism': 'human', 'tissue': ['colon', 'colorectum'], 'n_obs__lte': 400000}
include_uids: ['byYhE703lPTspj8i0000']
exclude_uids: None
families: ['spot', 'imaging']
marker_set: colon
rationale: Normal colon and colorectal cancer on Visium, Visium HD and Xenium; Oliveira 2025 was the mixing spike, here joined by GEO and 10x sections.

show_call(
    load_cohort,
    "Runs the query on the catalog, adds the included uids, removes the excluded ones, and loads each kept "
    "member's table keyed by gene symbol, joined within its technology family.",
)
adatas, cohort = load_cohort(spec, catalog=catalog)

columns = ["uid", "title", "technology", "family", "paper", "paper_source", "license", "n_obs", "status"]
show_table(cohort[columns].rename(columns={"license": "licence"}))
show_table(cohort["status"].value_counts().rename_axis("status").reset_index(name="members"))
print(f"{int((cohort['status'] == 'kept').sum())} kept, {int((cohort['status'] != 'kept').sum())} dropped.")

59 kept, 1 dropped.

show_call(load_thresholds, "Reads the QC threshold groups from thresholds.yaml.")
thresholds = load_thresholds()
show_table(
    pd.DataFrame(
        [
            {
                "group": name,
                "technologies": ", ".join(g["technologies"]),
                "min_counts": g["min_counts"],
                "min_genes": g["min_genes"],
                "max_pct_mt": g.get("max_pct_mt"),
                "flag: min median counts": g["flag"]["min_median_counts"],
                "flag: min share kept": g["flag"]["min_share_kept"],
                "reason": g["reason"],
            }
            for name, g in thresholds.items()
        ]
    ).set_index("group")
)
print(f"MIN_GENES_FOR_QC = {MIN_GENES_FOR_QC}: QC, and every view, needs at least this many genes.")

MIN_GENES_FOR_QC = 50: QC, and every view, needs at least this many genes.

show_call(
    apply_qc,
    "Leaves out control probes, computes counts, genes and the mitochondrial share per observation, keeps "
    "the observations that pass their technology's filters, and describes each dataset.",
)
qcs = []
for fam in list(adatas):
    adatas[fam], q = apply_qc(adatas[fam], thresholds)
    qcs.append(q.assign(family=fam))
qc = pd.concat(qcs, ignore_index=True)
show_table(qc)

MAX_VIOLIN_DATASETS = 40
for fam, adata in adatas.items():
    obs = adata.obs[["dataset_uid", "total_counts", "n_genes"]].copy()
    sizes = obs["dataset_uid"].value_counts()
    if len(sizes) > MAX_VIOLIN_DATASETS:
        print(f"{fam}: {len(sizes)} datasets, so the {MAX_VIOLIN_DATASETS} largest are shown.")
        obs = obs[obs["dataset_uid"].isin(sizes.index[:MAX_VIOLIN_DATASETS])]
    obs["dataset_uid"] = obs["dataset_uid"].astype(str).astype("category")
    display(Markdown(f"Counts and genes per observation after QC, family `{fam}`."))
    sc.pl.violin(ad.AnnData(obs=obs), ["total_counts", "n_genes"], groupby="dataset_uid", rotation=90, stripplot=False)

spot: 54 datasets, so the 40 largest are shown.

show_call(load_markers, "Reads the marker sets from markers.yaml, each with its citation and fallbacks.")
marker_set = load_markers()[spec["marker_set"]]
print(f"Marker set {spec['marker_set']!r}, cited from {', '.join(marker_set['citation'])}.")
show_table(
    pd.DataFrame({"markers": {c: ", ".join(g) for c, g in marker_set["compartments"].items()}}).rename_axis(
        "compartment"
    )
)
kept_after_qc = set(qc.loc[qc["status"] == "kept", "dataset_uid"].astype(str))
technology_of = cohort.set_index("uid")["technology"].astype(str)
panel_of = {
    uid: panel_id(technology_of[uid], genes)
    for adata in adatas.values()
    for uid, genes in adata.uns.get("member_genes", {}).items()
    if uid in kept_after_qc
}
used = [
    {
        "panel": pid,
        "compartment": comp,
        "genes": "not measured" if fb.get("not_measured") else ", ".join(fb.get("genes") or []),
        "note": fb.get("note", ""),
    }
    for pid, entries in (marker_set.get("fallback") or {}).items()
    if pid in set(panel_of.values())
    for comp, fb in entries.items()
]
if used:
    show_table(pd.DataFrame(used))
else:
    print("No fallback of this marker set applies to the panels of this cohort.")

Marker set 'colon', cited from 10.1016/j.cell.2019.06.029.

show_call(with_qc_status, "Marks the members that have no cells left after QC as dropped.")
kept_members = with_qc_status(cohort, qc).query("status == 'kept'")
ENSEMBL = r"ENS[A-Z]*G\d{6}"
SYMBOL_COLUMNS = ("symbol", "gene_symbols", "gene_symbol", "gene_name", "feature_name")


def colour_gene(table, panel):
    """(var name, symbol) of the first gene of the first compartment the table measures.

    A compartment's genes are its markers, then the fallback genes for this section's own panel.
    """
    names = pd.Index(table.var_names.astype(str))
    symbols = names
    if names.str.match(ENSEMBL).mean() >= 0.5:
        column = next((c for c in SYMBOL_COLUMNS if c in table.var), None)
        if column is None:
            return None, None
        symbols = pd.Index(table.var[column].astype(str))
    by_symbol = {s.upper(): (n, s) for n, s in zip(names, symbols, strict=True)}
    fallback = (marker_set.get("fallback") or {}).get(panel) or {}
    for comp, genes in marker_set["compartments"].items():
        for gene in [*genes, *((fallback.get(comp) or {}).get("genes") or [])]:
            if gene.upper() in by_symbol:
                return by_symbol[gene.upper()]
    return None, None


def table_region(zarr_url, table_name):
    """The shapes element a table annotates, read from the table's zarr attributes without loading it."""
    group = zarr.open_group(zarr_url, mode="r", storage_options=storage_options(zarr_url) or None)
    attrs = dict(group[f"tables/{table_name}"].attrs)
    if "region" not in attrs and f"tables/{table_name}/uns/spatialdata_attrs" in group:
        attrs = ad.io.read_elem(group[f"tables/{table_name}/uns/spatialdata_attrs"])
    region = attrs.get("region")
    return region if isinstance(region, str) or region is None else (list(region) or [None])[0]


def representative_uids(members):
    """The ``representatives`` kept sections closest to the median number of observations."""
    n_obs = members.set_index("uid")["n_obs"].astype(float)
    return list((n_obs - n_obs.median()).abs().sort_values(kind="stable").index[:representatives])

for tech, members in kept_members.groupby("technology"):
    for uid in representative_uids(members):
        ds = catalog.dataset(uid)
        stored = ds.elements()
        table_name = hd_table(stored) if tech == "VisiumHD" else (getattr(ds, "default_table", None) or "table")
        region = table_region(ds.zarr_url, table_name)
        if region is None or f"shapes/{region}" not in stored:
            display(Markdown(f"{tech}, section `{uid}`: the table annotates no shapes element, so no plot."))
            continue
        # a small tissue image when there is one: lowres, else hires (never the full-resolution image)
        images = [k.split("/", 1)[1] for k in stored if k.startswith("images/")]
        lowres = next((k for k in images if "lowres" in k), None) or next((k for k in images if "hires" in k), None)
        wanted = [f"tables/{table_name}", f"shapes/{region}"] + ([f"images/{lowres}"] if lowres else [])
        sdata = ds.to_sdata(elements=wanted)
        gene, symbol = colour_gene(sdata[table_name], panel_of.get(uid))
        section = members.set_index("uid").loc[uid, "title"]
        title = f"{tech}, {section}: {symbol}" if gene else f"{tech}, {section}"
        display(Markdown(f"{tech}, section `{uid}` ({int(members.set_index('uid').loc[uid, 'n_obs'])} observations)."))
        systems = list(get_transformation(sdata.shapes[region], get_all=True))
        system = "global" if "global" in systems else systems[0]
        # spatialdata-plot 0.4.2 rasterises shapes with datashader in "global" only; elsewhere use matplotlib
        method = None if system == "global" else "matplotlib"
        # fmt: off
        if lowres:
            (
                sdata
                .pl.render_images(lowres)
                .pl.render_shapes(region, color=gene, table_name=table_name, method=method, colorbar_params={"label": symbol})
                .pl.show(
                    coordinate_systems=system,
                    title=title,
                )
            )
        else:
            (
                sdata
                .pl.render_shapes(region, color=gene, table_name=table_name, method=method, colorbar_params={"label": symbol})
                .pl.show(
                    coordinate_systems=system,
                    title=title,
                )
            )
        # fmt: on

2026-11-04 05:11:05 | [INFO] root_attr: version

2026-11-04 05:11:05 | [INFO] root_attr: multiscales

2026-11-04 05:11:05 | [INFO] root_attr: omero

2026-11-04 05:11:05 | [INFO] datasets [{'path': 's0', 'coordinateTransformations': [{'type': 'scale', 'scale': [1.0, 1.0, 1.0]}, {'type': 'translation', 'translation': [0.0, 0.0, 0.0]}]}]

2026-11-04 05:11:05 | [INFO] resolution: s0

2026-11-04 05:11:05 | [INFO]  - shape ('c', 'y', 'x') = (3, 600, 600)

2026-11-04 05:11:05 | [INFO]  - chunks =  ['3', '600', '600']

2026-11-04 05:11:05 | [INFO]  - dtype = uint8

2026-11-04 05:11:05 | [INFO] root_attr: version

2026-11-04 05:11:05 | [INFO] root_attr: multiscales

show_call(
    qc_floors, "Gives each technology's count floor, the min_counts of its QC group, for gate 3 and the joint view."
)
floors = qc_floors(thresholds)
show_table(
    pd.DataFrame(
        {
            "value": [str(x) for x in (MIN_SHARED_GENES, MAX_FLOOR_LOSS, N_HVG, N_PCS, N_NEIGHBORS, SEED)],
            "meaning": [
                "genes a view needs before a family is split per technology, and the joint view's gate 1",
                "largest share of a technology's observations that may fall below its floor on the shared genes (gate 3)",
                "highly variable genes kept when a view has more genes",
                "principal components",
                "neighbours per observation",
                "random seed",
            ],
        },
        index=["MIN_SHARED_GENES", "MAX_FLOOR_LOSS", "N_HVG", "N_PCS", "N_NEIGHBORS", "SEED"],
    )
)
show_table(pd.Series(floors, name="count floor").rename_axis("technology").to_frame())

show_call(
    embed_views,
    "Builds one view per family (or per technology when a family shares too few genes), plus the joint view "
    "when the gates pass, and embeds each with PCA, UMAP and Harmony on the section.",
)
views = embed_views(adatas, marker_set, floor=floors, use_gpu=use_gpu)
for name, v in views.items():
    p = v.uns["primer"]
    print(f"{name}: {p['n_obs']} observations, {p['n_sections']} sections; {p['genes']['rule']}. {p['note']}".strip())
first = next(iter(views.values())).uns["primer"]
for uid, reason in first["members_dropped_panel"].items():
    print(f"member {uid} {reason}")
for c in {tuple(c["uids"]): c for v in views.values() for c in v.uns["primer"]["panel_costs"]}.values():
    print(f"{c['technology']} panel of {', '.join(c['uids'])}: {c['reason']} shared genes")
for name, reason in first["views_skipped"].items():
    print(f"no view {name}: {reason}")
gates = next((v.uns["primer"]["gates"] for v in views.values() if "gates" in v.uns["primer"]), None)
if gates is not None:
    print(f"Joint view: {'yes' if gates['joint'] else 'no'}. {gates['reason']}".strip())
    for drop in gates["drops"]:
        print(f"  left out {drop['technology']} ({drop['step']}): {drop['reason']}")

spot: 441580 observations, 54 sections; 2000 highly variable genes of the 17367 measured, expressed genes (seurat_v3 on raw counts, section as batch); its 6 gene panels have 17,943 to 19,144 genes each and share 17,368.
imaging_cell: 1429427 observations, 5 sections; all 322 measured, expressed genes of the view; its 3 gene panels have 325 to 425 genes each and share 322.
Joint view: no. fewer than two families keep a technology
  left out Visium (gate3): loses 71% of its observations below 500 counts on the 299 shared genes (limit 50%)
  left out VisiumHD (gate3): loses 99% of its observations below 200 counts on the 299 shared genes (limit 50%)

for name, v in views.items():
    colours = [c for c in ("paper", "technology", "tissue") if c in v.obs and v.obs[c].nunique() > 1]
    display(Markdown(f"View `{name}`, before Harmony."))
    sc.pl.umap(v, color=colours or None, ncols=1)
    if "X_umap_harmony" in v.obsm:
        display(Markdown(f"View `{name}`, after Harmony on the section."))
        sc.pl.embedding(v, basis="X_umap_harmony", color=colours or None, ncols=1)
    else:
        display(Markdown(f"View `{name}` has one section, so it has no Harmony embedding."))

show_call(
    mixing_by_view,
    "Measures how evenly the batches mix among each observation's nearest neighbours, per view, before and "
    "after Harmony; the bundle's manifest records the same numbers.",
)
mixing = mixing_by_view(views)
rows, definitions = [], {}
for name, by_batch in mixing.items():
    for label, by_basis in by_batch.items():
        for basis, m in by_basis.items():
            for stat in ("entropy", "other_share"):
                definitions.setdefault(stat, m[stat]["definition"].replace(m["batch_key"], "batch"))
                rows.append(
                    {
                        "view": name,
                        "batch": label,
                        "embedding": "after Harmony" if basis == "X_pca_harmony" else "before Harmony",
                        "statistic": stat,
                        "mean": m[stat]["mean"],
                        "random reference": m[stat]["random"],
                        "limit from proportions": m[stat]["random_limit"],
                        "note": m["note"],
                    }
                )
show_table(pd.DataFrame(rows))
for stat, text in definitions.items():
    print(f"{stat}: {text}")
print(CAVEAT)

entropy: How evenly the batch labels are spread among each observation's 30 nearest neighbours, divided by log(min(30, number of labels)): 0 when all neighbours share one label, 1 when the labels are as evenly spread as the neighbours allow.
other_share: The share of each observation's 30 nearest neighbours that carry another batch.
Well mixed is not the same as biologically right: a correction can also mix cells that differ for a real reason, such as tissue or disease.

fallback_genes = {}
for entries in (marker_set.get("fallback") or {}).values():
    for comp, fb in (entries or {}).items():
        fallback_genes.setdefault(comp, set()).update(str(g).upper() for g in (fb or {}).get("genes") or [])
compartment_genes = {
    comp: {str(g).upper() for g in genes} | fallback_genes.get(comp, set())
    for comp, genes in marker_set["compartments"].items()
}
readings = {}
for name, v in views.items():
    key = "nn_harmony" if "nn_harmony_distances" in v.obsp else "nn_pca"
    leiden = {"resolution": 0.5, "random_state": SEED, "neighbors_key": key, "key_added": "leiden"}
    if use_gpu:  # rapids-singlecell's n_iterations counts aggregation levels, so its default is kept
        rsc.tl.leiden(v, **leiden)
    else:
        sc.tl.leiden(v, flavor="igraph", n_iterations=2, **leiden)
    expr = v.obsm["marker_expr"]
    m = ad.AnnData(
        X=expr.to_numpy(dtype=np.float32),
        obs=v.obs[["leiden"]].copy(),
        var=pd.DataFrame(index=expr.columns.astype(str)),
    )
    display(Markdown(f"View `{name}`: {m.obs['leiden'].nunique()} clusters on `{key}`, {m.n_vars} marker genes."))
    if m.n_vars == 0 or m.obs["leiden"].nunique() < 2:
        display(Markdown("This view has fewer than two clusters or no marker gene, so there is no dotplot."))
        continue
    by_compartment = {
        comp: [g for g in expr.columns if g.upper() in genes] for comp, genes in compartment_genes.items()
    }
    sc.pl.dotplot(m, {k: g for k, g in by_compartment.items() if g}, groupby="leiden", standard_scale="var")
    scores = (
        pd.DataFrame(
            {
                comp: expr[[g for g in expr.columns if g.upper() in genes]].mean(axis=1)
                for comp, genes in compartment_genes.items()
                if any(g.upper() in genes for g in expr.columns)
            }
        )
        .groupby(m.obs["leiden"].to_numpy())
        .mean()
    )
    ranked = scores.apply(lambda r: r.sort_values(ascending=False), axis=1)
    table = pd.DataFrame(
        {
            "observations": m.obs["leiden"].value_counts().reindex(scores.index),
            "top compartment": scores.idxmax(axis=1),
            "top mean": scores.max(axis=1),
            "runner-up": [r.dropna().index[1] if r.notna().sum() > 1 else "" for _, r in ranked.iterrows()],
            "runner-up mean": [r.dropna().iloc[1] if r.notna().sum() > 1 else np.nan for _, r in ranked.iterrows()],
        }
    ).rename_axis("cluster")
    readings[name] = table
    show_table(table)

for name, table in readings.items():
    lines = [f"View `{name}`:"]
    for cluster, r in table.iterrows():
        text = (
            f"- Cluster {cluster} ({int(r['observations'])} observations) has its highest mean marker expression "
            f"for {r['top compartment']} ({r['top mean']:.2f})"
        )
        if r["runner-up"]:
            text += f", followed by {r['runner-up']} ({r['runner-up mean']:.2f})"
        lines.append(text + ".")
    display(Markdown("\n".join(lines)))

show_call(
    export_bundle,
    "Writes the manifest, umap.parquet, qc.parquet, cohort.parquet and embedding.h5ad for this cohort. A cohort whose embedding is over 1 GiB gets parts of whole datasets instead (embedding-1.h5ad, embedding-2.h5ad, ...), and anndata.concat over the parts gives the whole cohort.",
)
bundle = export_bundle(
    views,
    cohort,
    qc,
    spec,
    out_dir,
    release,
    runtime_s=time.time() - start,
    mixing=mixing,
    primers_ref=primers_ref,
)
for f in sorted(bundle.iterdir()):
    print(f"{f.relative_to(out_dir)}  {f.stat().st_size} bytes")
manifest = json.loads((bundle / "manifest.json").read_text())
for key in (
    "slug",
    "release",
    "primers_ref",
    "n_datasets",
    "n_papers",
    "technologies",
    "n_cells_before",
    "n_cells_after",
    "n_flagged",
    "path",
    "hardware",
    "runtime_s",
):
    print(f"{key}: {manifest[key]}")

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

command = (
    f"papermill primers/notebooks/primer.ipynb primer.ipynb -p slug {slug} -p release {release} "
    f"-p primers_ref {primers_ref} -p use_gpu {use_gpu}" + (f" -p smoke {smoke}" if smoke else "")
)
versions = "\n".join(f"  - {k} {v}" for k, v in manifest["versions"].items())
display(
    Markdown(
        f"- Catalog release: `{release}`\n- primers reference: `{primers_ref}`\n- Versions:\n{versions}\n"
        f"- Hardware: {manifest['hardware']} ({platform.system()})\n- Path: {manifest['path']}\n\n"
        "To run this primer again:"
    )
)
show_command(command)

papermill primers/notebooks/primer.ipynb primer.ipynb -p slug human-colon -p release 2026-11-01 -p primers_ref 4d7af340849c00077f9d21c5b54af9e689a4a070 -p use_gpu True

	uid	title	technology	family	paper	paper_source	licence	n_obs	status
0	r0hI7AaM1crJtqq60001	visium-cytassist-gene-expression-libraries-of-post-xenium-human-colon-cancer-ffpe-using-the-human-whole-transcriptome-probe-set-2-standard	Visium	spot	10x Genomics demo datasets	10x_demo	CC-BY-4.0	6518	kept
1	4LGaWiMGNdmI79Si0002	V53M06-039	Visium	spot	visium-cytassist-gene-expression-of-post-xenium-vs-control	collection	CC-BY-4.0	6352	kept
2	2Z4uTqeIZtrYw0YY0000	GSM9322957	Visium	spot	geo-gse311294	geo_series	ncbi-geo-policy	3742	kept
3	LEmjbIthI5Ge9P640002	V53M06-039	Visium	spot	visium-cytassist-gene-expression-of-post-xenium-vs-control	collection	CC-BY-4.0	6487	kept
4	4gQynuR0sN7FY8zx0002	V53M06-039	Visium	spot	visium-cytassist-gene-expression-of-post-xenium-vs-control	collection	CC-BY-4.0	6414	kept
5	1Fn1jXIrYRqetnxm0000	human-colon-preview-data-xenium-human-colon-gene-expression-panel-1-standard__hcolon-non-diseased-add-on-ffpe	Xenium	imaging_cell	10x Genomics demo datasets	10x_demo	CC-BY-4.0	275822	kept
6	VAHI5ZX2Kd4z4xo70000	GSM9340364	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	11346	kept
7	nlnQtxRJaHzD4QGg0000	GSM9340372	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	11743	kept
8	3Kv4Urjh9wOvpWxx0000	GSM9340378	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	10493	kept
9	apO7dpGaSHFeGqV90000	GSM9340365	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	9022	kept
10	5k9x6zgPxGCqcVm90000	GSM9340371	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	8620	kept
11	bcFco58JPI1KGPxv0000	GSM9340381	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	11709	kept
12	hXUgLFFUrOsFU2cC0000	xenium-human-colon-cancer-p1-crc-addon	Xenium	imaging_cell	10.1038/s41588-025-02193-3	doi_inferred	CC-BY-4.0	307762	kept
13	oXs8e80rkMTec58P0000	GSM9340367	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	13362	kept
14	JghmzMkVMneMc4fc0000	GSM9340379	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	13643	kept
15	vQ7Ea8u6egcE5Yl80000	GSM9340377	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	11174	kept
16	K3y4WdsggesEJx9t0000	visium-cytassist-human-colon-normal-p5	Visium	spot	10.1038/s41588-025-02193-3	doi	CC-BY-4.0	2638	kept
17	IuKLRdW4rbmKcGaf0000	GSM9340370	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	9421	kept
18	V61FMJpIJmFzPcRh0000	GSM9340366	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	8851	kept
19	ETm3REX6qjG180pe0000	visium-cytassist-human-colon-normal-p3	Visium	spot	10.1038/s41588-025-02193-3	doi	CC-BY-4.0	3887	kept
20	AbWx6fWqvaHdq2Ng0000	GSM9340380	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	13022	kept
21	Hryzjbyxm1LVVuxm0000	GSM6505135	Visium	spot	10.1016/j.crmeth.2022.100340	doi	ncbi-geo-policy	2261	kept
22	usUHxzCZfpKtlsGe0000	GSM9340363	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	7269	kept
23	Ewx754a3yUZfDGAe0000	GSM9340375	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	14336	kept
24	qMnnMcP2B2W5ddYN0000	visium-cytassist-human-colon-cancer-p2	Visium	spot	10.1038/s41588-025-02193-3	doi	CC-BY-4.0	4269	kept
25	cvzmUpPy4joWqoLn0000	GSM9340373	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	8076	kept
26	OANiWcfU8xRy95iN0000	GSM9340368	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	7431	kept
27	bZvfnosh4QOGK1WJ0000	GSM9340376	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	7139	kept
28	Q8iE0HqeTmMZDAPU0000	GSM9340369	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	11842	kept
29	p2dczaEpp7jiUMaC0000	GSM9340374	Visium	spot	10.1038/s41467-026-71944-5	doi	CC-BY-4.0	9890	kept
30	ODlV9hhd8502waGR0000	xenium-human-colon-cancer-p2-crc-addon	Xenium	imaging_cell	10.1038/s41588-025-02193-3	doi_inferred	CC-BY-4.0	340837	kept
31	HOoZeyrVaBDC3qRR0000	xenium-human-colon-cancer-p5-crc-addon	Xenium	imaging_cell	10.1038/s41588-025-02193-3	doi_inferred	CC-BY-4.0	275998	kept
32	fsVBVE8Vt4tYAbnL0000	GSM6505133	Visium	spot	10.1016/j.crmeth.2022.100340	doi	ncbi-geo-policy	2204	kept
33	FkH5OwJmiFgtKYsu0000	GSM6505134	Visium	spot	10.1016/j.crmeth.2022.100340	doi	ncbi-geo-policy	3232	kept
34	qcge4iApT3jyYAmz0000	GSM9785433	Visium	spot	geo-gse334323	geo_series	ncbi-geo-policy	4898	kept
35	rDBWjvddZYTELxem0000	GSM9322960	Visium	spot	geo-gse311294	geo_series	ncbi-geo-policy	4362	kept
36	xHInB8UtsIi4YsFm0000	human-colorectal-cancer-11-mm-capture-area-ffpe-2-standard	Visium	spot	10x Genomics demo datasets	10x_demo	CC-BY-4.0	9080	kept
37	FUV4muFggCuWi4US0000	GSM9554207	Visium	spot	10.1126/science.ady1678	doi	ncbi-geo-policy	3579	kept
38	9Q3xbJk09P0dcb5S0000	GSM9322959	Visium	spot	geo-gse311294	geo_series	ncbi-geo-policy	3673	kept
39	ASZTbVCdOyoFEfXH0000	GSM8703563	Visium	spot	10.1038/s41467-025-66485-2	doi	ncbi-geo-policy	3493	kept
40	WIs54w6nWccKpx310000	GSM8703566	Visium	spot	10.1038/s41467-025-66485-2	doi	ncbi-geo-policy	3415	kept
41	50NMBRnIt2dhttPu0000	GSM8703565	Visium	spot	10.1038/s41467-025-66485-2	doi	ncbi-geo-policy	3563	kept
42	rmy1EEWRM1RZvdyX0000	GSM9785428	Visium	spot	geo-gse334323	geo_series	ncbi-geo-policy	4992	kept
43	ekNMyKACI3IvML1s0000	GSM9554205	Visium	spot	10.1126/science.ady1678	doi	ncbi-geo-policy	2946	kept
44	XomWzX9g1eBUKuFm0000	GSM9322958	Visium	spot	geo-gse311294	geo_series	ncbi-geo-policy	4732	kept
45	ETWGEM211qDYxYxR0000	GSM7089855	Visium	spot	geo-gse226997	geo_series	ncbi-geo-policy	4148	kept
46	GEyvRPhzNhAuMQ1m0000	GSM9785429	Visium	spot	geo-gse334323	geo_series	ncbi-geo-policy	4682	kept
47	XObX8m20iIl0l2A30000	GSM9554206	Visium	spot	10.1126/science.ady1678	doi	ncbi-geo-policy	2599	kept
48	afvnxtewYmMuUsLE0000	GSM9785430	Visium	spot	geo-gse334323	geo_series	ncbi-geo-policy	3689	kept
49	0YPANyxcDKJmaAQM0000	GSM7089857	Visium	spot	geo-gse226997	geo_series	ncbi-geo-policy	4676	kept
50	JQBMUJ2pwE4VrkcE0000	GSM7089858	Visium	spot	geo-gse226997	geo_series	ncbi-geo-policy	3535	kept
51	ClgKsG6HK1dBEMby0000	GSM7089856	Visium	spot	geo-gse226997	geo_series	ncbi-geo-policy	3737	kept
52	HGIUBMBQQA0YI0ap0000	GSM8703564	Visium	spot	10.1038/s41467-025-66485-2	doi	ncbi-geo-policy	2473	kept
53	iidMPcbVlfusWRbP0001	ffpe-human-colorectal-cancer-data-with-human-immuno-oncology-profiling-panel-and-custom-add-on-1-standard	Xenium	imaging_cell	n/a	none	CC-BY-4.0	388175	dropped: no paper (no study_id, series or collection)
54	mxS1cBFjzkguZFvq0000	GSM9785432	Visium	spot	geo-gse334323	geo_series	ncbi-geo-policy	3565	kept
55	tBcLc5Z5NkufJCMS0000	GSM9785431	Visium	spot	geo-gse334323	geo_series	ncbi-geo-policy	3909	kept
56	ZcUlrf9ps0SifYJc0001	human-colon-preview-data-xenium-human-colon-gene-expression-panel-1-standard__hcolon-non-diseased-base-ffpe	Xenium	imaging_cell	10x Genomics demo datasets	10x_demo	CC-BY-4.0	270984	kept
57	l4gygKGF3CVWnkN60000	GSM9322961	Visium	spot	geo-gse311294	geo_series	ncbi-geo-policy	4564	kept
58	PoO79PXNXVvmgaQ10000	GSM9554204	Visium	spot	10.1126/science.ady1678	doi	ncbi-geo-policy	2340	kept
59	byYhE703lPTspj8i0000	visium-hd-human-colon-cancer-p2	VisiumHD	spot	10.1038/s41588-025-02193-3	doi_inferred	CC-BY-4.0	545913	kept

	dataset_uid	technology	n_before	n_after	median_counts	median_genes	pct_mt_median	flagged	flag_reason	status	family
0	0YPANyxcDKJmaAQM0000	Visium	4676	4508	7569.0	3475.5	0.0	False		kept	spot
1	2Z4uTqeIZtrYw0YY0000	Visium	3742	3582	3885.5	2541.5	14.808	False		kept	spot
2	3Kv4Urjh9wOvpWxx0000	Visium	10493	10490	4596.0	2998.0	3.205	False		kept	spot
3	4LGaWiMGNdmI79Si0002	Visium	6352	5009	6810.5	3015.5	1.97	False		kept	spot
4	4gQynuR0sN7FY8zx0002	Visium	6414	5325	4009.5	2403.5	4.427	False		kept	spot
5	50NMBRnIt2dhttPu0000	Visium	3563	3555	4687.0	2505.0	0.0	False		kept	spot
6	5k9x6zgPxGCqcVm90000	Visium	8620	8086	4980.0	2866.0	9.59	False		kept	spot
7	9Q3xbJk09P0dcb5S0000	Visium	3673	3453	2020.0	1548.0	5.198	False		kept	spot
8	ASZTbVCdOyoFEfXH0000	Visium	3493	3491	4129.0	2376.0	0.0	False		kept	spot
9	AbWx6fWqvaHdq2Ng0000	Visium	13022	13018	14744.5	5654.0	2.778	False		kept	spot
10	ClgKsG6HK1dBEMby0000	Visium	3737	3588	4436.0	2474.0	0.0	False		kept	spot
11	ETWGEM211qDYxYxR0000	Visium	4148	4142	15059.0	5543.5	0.0	False		kept	spot
12	ETm3REX6qjG180pe0000	Visium	3887	3186	1920.0	1414.0	3.793	False		kept	spot
13	Ewx754a3yUZfDGAe0000	Visium	14336	14066	24268.0	7352.5	4.235	False		kept	spot
14	FUV4muFggCuWi4US0000	Visium	3579	3558	8213.0	3787.0	0.0	False		kept	spot
15	FkH5OwJmiFgtKYsu0000	Visium	3232	3010	2281.5	1254.5	0.0	False		kept	spot
16	GEyvRPhzNhAuMQ1m0000	Visium	4682	2131	357.5	295.0	8.174	True	median counts 357.5 < 1000; 46% kept < 50%	kept	spot
17	HGIUBMBQQA0YI0ap0000	Visium	2473	2442	1547.0	1114.0	0.0	False		kept	spot
18	Hryzjbyxm1LVVuxm0000	Visium	2261	2137	2239.0	1243.0	0.0	False		kept	spot
19	IuKLRdW4rbmKcGaf0000	Visium	9421	9415	11141.0	4945.0	3.151	False		kept	spot
20	JQBMUJ2pwE4VrkcE0000	Visium	3535	3264	2466.0	1675.0	0.0	False		kept	spot
21	JghmzMkVMneMc4fc0000	Visium	13643	13638	20542.0	6851.0	3.136	False		kept	spot
22	K3y4WdsggesEJx9t0000	Visium	2638	2623	10514.5	4689.5	6.75	False		kept	spot
23	LEmjbIthI5Ge9P640002	Visium	6487	5926	5464.0	3018.0	4.815	False		kept	spot
24	OANiWcfU8xRy95iN0000	Visium	7431	7402	6662.0	3510.0	4.129	False		kept	spot
25	PoO79PXNXVvmgaQ10000	Visium	2340	2338	6743.5	3155.0	0.0	False		kept	spot
26	Q8iE0HqeTmMZDAPU0000	Visium	11842	11206	2462.5	1800.0	7.762	False		kept	spot
27	V61FMJpIJmFzPcRh0000	Visium	8851	8378	21274.0	6302.0	2.922	False		kept	spot
28	VAHI5ZX2Kd4z4xo70000	Visium	11346	11340	6960.0	3674.5	4.331	False		kept	spot
29	WIs54w6nWccKpx310000	Visium	3415	3405	3502.0	1953.0	0.0	False		kept	spot
30	XObX8m20iIl0l2A30000	Visium	2599	2563	3550.0	2214.0	0.0	False		kept	spot
31	XomWzX9g1eBUKuFm0000	Visium	4732	4623	4920.0	3066.5	12.603	False		kept	spot
32	afvnxtewYmMuUsLE0000	Visium	3689	3653	4457.0	2609.0	3.603	False		kept	spot
33	apO7dpGaSHFeGqV90000	Visium	9022	9012	4021.5	2540.0	6.495	False		kept	spot
34	bZvfnosh4QOGK1WJ0000	Visium	7139	6825	12746.0	5453.0	3.372	False		kept	spot
35	bcFco58JPI1KGPxv0000	Visium	11709	11707	12136.0	5739.0	4.83	False		kept	spot
36	byYhE703lPTspj8i0000	VisiumHD	137051	121249	1598.0	1071.0	4.049	False		kept	spot
37	cvzmUpPy4joWqoLn0000	Visium	8076	8075	11518.5	4829.5	3.021	False		kept	spot
38	ekNMyKACI3IvML1s0000	Visium	2946	2941	14731.0	5172.5	0.0	False		kept	spot
39	fsVBVE8Vt4tYAbnL0000	Visium	2204	2204	30740.0	7510.5	0.0	False		kept	spot
40	l4gygKGF3CVWnkN60000	Visium	4564	4120	1256.0	969.5	14.643	False		kept	spot
41	mxS1cBFjzkguZFvq0000	Visium	3565	3165	3687.0	2260.0	4.819	False		kept	spot
42	nlnQtxRJaHzD4QGg0000	Visium	11743	11740	26371.0	7512.0	5.782	False		kept	spot
43	oXs8e80rkMTec58P0000	Visium	13362	13362	30463.0	7685.5	3.254	False		kept	spot
44	p2dczaEpp7jiUMaC0000	Visium	9890	9878	6312.5	3586.5	3.659	False		kept	spot
45	qMnnMcP2B2W5ddYN0000	Visium	4269	4269	75589.0	11383.0	3.156	False		kept	spot
46	qcge4iApT3jyYAmz0000	Visium	4898	4578	1986.5	1140.5	3.738	False		kept	spot
47	r0hI7AaM1crJtqq60001	Visium	6518	4930	3374.0	1974.5	2.267	False		kept	spot
48	rDBWjvddZYTELxem0000	Visium	4362	4267	1541.5	1218.0	6.546	False		kept	spot
49	rmy1EEWRM1RZvdyX0000	Visium	4992	674	142.0	125.0	7.273	True	median counts 142 < 1000; 14% kept < 50%	kept	spot
50	tBcLc5Z5NkufJCMS0000	Visium	3909	2776	1181.0	850.0	10.836	False		kept	spot
51	usUHxzCZfpKtlsGe0000	Visium	7269	7009	3837.0	2347.0	6.58	False		kept	spot
52	vQ7Ea8u6egcE5Yl80000	Visium	11174	11171	10762.0	5502.5	4.963	False		kept	spot
53	xHInB8UtsIi4YsFm0000	Visium	9080	9077	49281.5	9559.5	4.387	False		kept	spot
54	1Fn1jXIrYRqetnxm0000	Xenium	275822	274239	84.0	41.0	n/a	False		kept	imaging_cell
55	HOoZeyrVaBDC3qRR0000	Xenium	275998	270955	80.0	41.0	n/a	False		kept	imaging_cell
56	ODlV9hhd8502waGR0000	Xenium	340837	331376	94.0	50.0	n/a	False		kept	imaging_cell
57	ZcUlrf9ps0SifYJc0001	Xenium	270984	263505	53.0	31.0	n/a	False		kept	imaging_cell
58	hXUgLFFUrOsFU2cC0000	Xenium	307762	289352	53.0	32.0	n/a	False		kept	imaging_cell

	markers
compartment
epithelial	EPCAM, KRT20, CDH17, CEACAM5
goblet	MUC2, TFF3, SPINK4
stem	LGR5, OLFM4, ASCL2
fibroblast	COL1A1, COL3A1, DCN, PDGFRA
smooth_muscle	ACTA2, DES, MYH11
endothelial	PECAM1, VWF, PLVAP
t_cell	CD3E, CD3D, CD2
b_plasma	MS4A1, CD79A, MZB1, JCHAIN
myeloid	CD68, CD14, CSF1R, C1QA

	panel	compartment	genes	note
0	Xenium-425-4d5186a6	goblet	CLCA1, SPDEF, ITLN1	goblet-cell genes on this panel
1	Xenium-422-21ae9d3d	goblet	CLCA1, SPDEF, ITLN1	goblet-cell genes on this panel
2	Xenium-325-2fc53225	goblet	CLCA1, SPDEF, ITLN1	goblet-cell genes on this panel
3	Xenium-325-2fc53225	fibroblast	not measured	no fibroblast gene on this panel
4	Xenium-325-2fc53225	smooth_muscle	not measured	no smooth-muscle gene on this panel
5	Xenium-325-2fc53225	endothelial	AQP1	the only endothelial gene on this panel; AQP1 is also in some epithelia, so weak alone

	count floor
technology
Visium	500.0
SpatialTranscriptomics	500.0
VisiumHD	200.0
Xenium	10.0
Merscope	10.0
MERFISH	10.0

Cohort primer¶

Human colon and colorectal cancer¶

Setup¶

Query and cohort¶

Quality control¶

Marker set¶

Spatial overview¶

Embedding¶

Biology¶

Draft reading, to be reviewed by the Lead¶

Export and reproduce¶

	technologies	min_counts	min_genes	max_pct_mt	flag: min median counts	flag: min share kept	reason
group
spot	Visium, SpatialTranscriptomics	500	250	30.0	1000	0.5	A Visium or ST spot holds about 1 to 10 cells, so a spot with under 500 UMIs or 250 genes is mostly background or tissue edge; solid tissues often reach 20 to 30% mitochondrial reads, so the cap is 30%.
binned_spot	VisiumHD	200	50	30.0	200	0.5	On the 5 Visium HD sections of oliveira-2024-colorectal, 16 um bins at 200 counts kept 74% of bins with the same marker signal as 8 um bins at Oliveira et al.'s 100 (Nat Genet 57:1512-1523), which kept 60%, and found T cells in all 5 sections against 3 (web tasks/2026-11-01-t094-hd-bin-size.md).
imaging_cell	Xenium, Merscope, MERFISH	10	3	n/a	30	0.5	Targeted panels give tens to hundreds of decoded transcripts per segmented cell, and a cell with under 10 transcripts or 3 genes cannot be given a type; the panels have no mitochondrial genes.

	value	meaning
MIN_SHARED_GENES	200	genes a view needs before a family is split per technology, and the joint view's gate 1
MAX_FLOOR_LOSS	0.5	largest share of a technology's observations that may fall below its floor on the shared genes (gate 3)
N_HVG	2000	highly variable genes kept when a view has more genes
N_PCS	30	principal components
N_NEIGHBORS	30	neighbours per observation
SEED	0	random seed

	view	batch	embedding	statistic	mean	random reference	limit from proportions
0	spot	section	before Harmony	entropy	0.09	0.756	0.983
1	spot	section	before Harmony	other_share	0.146	0.881	0.911
2	spot	section	after Harmony	entropy	0.386	0.755	0.983
3	spot	section	after Harmony	other_share	0.565	0.881	0.911
4	spot	paper	before Harmony	entropy	0.033	0.621	0.695
5	spot	paper	before Harmony	other_share	0.041	0.682	0.706
6	spot	paper	after Harmony	entropy	0.188	0.621	0.695
7	spot	paper	after Harmony	other_share	0.224	0.682	0.706
8	imaging_cell	section	before Harmony	entropy	0.467	0.955	0.998
9	imaging_cell	section	before Harmony	other_share	0.415	0.772	0.799
10	imaging_cell	section	after Harmony	entropy	0.658	0.955	0.998
11	imaging_cell	section	after Harmony	other_share	0.579	0.772	0.799
12	imaging_cell	paper	before Harmony	entropy	0.291	0.93	0.955
13	imaging_cell	paper	before Harmony	other_share	0.119	0.454	0.469
14	imaging_cell	paper	after Harmony	entropy	0.467	0.93	0.955
15	imaging_cell	paper	after Harmony	other_share	0.213	0.454	0.469

	observations	top compartment	top mean	runner-up	runner-up mean
cluster
0	22820	fibroblast	1.654	endothelial	0.323
1	12974	epithelial	1.601	endothelial	0.095
10	71447	epithelial	1.248	endothelial	0.122
11	924	fibroblast	1.946	endothelial	0.558
12	45416	goblet	0.938	endothelial	0.225
2	8631	fibroblast	0.938	endothelial	0.155
3	79383	epithelial	1.353	endothelial	0.07
4	46230	epithelial	2.167	endothelial	0.095
5	27127	fibroblast	1.638	endothelial	0.437
6	62358	fibroblast	1.935	endothelial	0.24
7	35779	smooth_muscle	2.916	endothelial	0.286
8	25250	fibroblast	1.944	endothelial	0.72
9	3241	fibroblast	0.679	endothelial	0.098

	observations	top compartment	top mean	runner-up	runner-up mean
cluster
0	114318	myeloid	0.457	endothelial	0.056
1	33954	epithelial	0.139	endothelial	0.073
10	39317	b_plasma	1.218	endothelial	0.029
11	116323	epithelial	0.955	endothelial	0.033
12	79188	epithelial	1.359	endothelial	0.002
13	1897	endothelial	0.192	endothelial	0.192
14	61	goblet	0.029	endothelial	0.0
15	43	epithelial	0.167	endothelial	0.0
2	77661	b_plasma	0.478	endothelial	0.043
3	248111	endothelial	0.15	endothelial	0.15
4	160326	t_cell	0.458	endothelial	0.043
5	22846	epithelial	0.381	endothelial	0.036
6	248936	epithelial	1.098	endothelial	0.043
7	118499	epithelial	0.62	endothelial	0.045
8	112140	epithelial	0.575	endothelial	0.133
9	55807	endothelial	1.117	endothelial	1.117