slug = "human-colon"
release = "2026-10-28"
use_gpu = True
out_dir = "bundles"
smoke = 0
primers_ref = "feat-primers"
cohorts_path = None
catalog_path = None
representatives = 1
dev_note = ""

# Parameters
slug = "mouse-kidney"
release = "2026-11-01"
out_dir = "bundles"
primers_ref = "4d7af340849c00077f9d21c5b54af9e689a4a070"
dev_note = ""

pip install "sddb-primers @ git+https://github.com/theislab/spatialdata-db-client@4d7af340849c00077f9d21c5b54af9e689a4a070#subdirectory=primers"

import inspect
import json
import platform
import time

import anndata as ad
import numpy as np
import pandas as pd
import scanpy as sc
import spatialdata_plot  # noqa: F401  (registers the .pl accessor)
import zarr
from primers.cohort import hd_table, load_cohort, smoke_spec
from primers.embed import CAVEAT, MAX_FLOOR_LOSS, MIN_SHARED_GENES, N_HVG, N_NEIGHBORS, N_PCS, SEED, embed_views
from primers.export import export_bundle, mixing_by_view, with_qc_status
from primers.markers import load_markers, panel_id
from primers.qc import MIN_GENES_FOR_QC, apply_qc, load_thresholds, qc_floors
from spatialdata.transformations import get_transformation

import sddb
from sddb.remote import storage_options

start = time.time()
out_dir = Path(out_dir)
if use_gpu:
    import cupy as cp
    import rapids_singlecell as rsc
    import rmm
    from rmm.allocators.cupy import rmm_cupy_allocator

    rmm.reinitialize(managed_memory=True)
    cp.cuda.set_allocator(rmm_cupy_allocator)


def show_call(func, what):
    """Show what a primers function does and link its source at ``primers_ref``."""
    _, line = inspect.getsourcelines(func)
    module = func.__module__.split(".")[-1]
    url = f"{REPO}/blob/{primers_ref}/primers/primers/{module}.py#L{line}"
    display(Markdown(f"`{func.__module__}.{func.__name__}`: {what} [Source at `{primers_ref}`]({url})."))


def show_table(df, digits=3):
    """Display ``df`` rounded, with missing values written as n/a, and long text columns in full."""
    shown = df.round(digits).astype(object)
    with pd.option_context("display.max_colwidth", None):
        display(shown.where(df.notna(), "n/a"))


catalog = sddb.Catalog.from_file(catalog_path) if catalog_path else sddb.Catalog(version=release)

if smoke:
    show_call(
        smoke_spec,
        "Keeps only the smallest kept member of each of a few papers, taken in turn from each family, for a "
        "quick end-to-end check.",
    )
    n_spec_excluded = len(spec.get("exclude_uids") or [])
    spec = smoke_spec(spec, catalog, smoke)
    print(
        f"Smoke run on {smoke} papers: {len(spec['exclude_uids']) - n_spec_excluded} more uids are excluded "
        f"({len(spec['exclude_uids'])} in all)."
    )
for key in ("slug", "query", "include_uids", "exclude_uids", "families", "marker_set", "rationale"):
    value = spec.get(key)
    if smoke and key == "exclude_uids":
        value = f"{len(value)} uids"
    print(f"{key}: {value}")

slug: mouse-kidney
query: {'organism': 'mouse', 'tissue': 'kidney', 'n_obs__lte': 450000}
include_uids: None
exclude_uids: None
families: ['spot', 'imaging']
marker_set: kidney
rationale: Mouse kidney injury and fibrosis on Visium, Visium HD and Xenium; the cap leaves out the two largest HD sections.

show_call(
    load_cohort,
    "Runs the query on the catalog, adds the included uids, removes the excluded ones, and loads each kept "
    "member's table keyed by gene symbol, joined within its technology family.",
)
adatas, cohort = load_cohort(spec, catalog=catalog)

columns = ["uid", "title", "technology", "family", "paper", "paper_source", "license", "n_obs", "status"]
show_table(cohort[columns].rename(columns={"license": "licence"}))
show_table(cohort["status"].value_counts().rename_axis("status").reset_index(name="members"))
print(f"{int((cohort['status'] == 'kept').sum())} kept, {int((cohort['status'] != 'kept').sum())} dropped.")

59 kept, 5 dropped.

show_call(load_thresholds, "Reads the QC threshold groups from thresholds.yaml.")
thresholds = load_thresholds()
show_table(
    pd.DataFrame(
        [
            {
                "group": name,
                "technologies": ", ".join(g["technologies"]),
                "min_counts": g["min_counts"],
                "min_genes": g["min_genes"],
                "max_pct_mt": g.get("max_pct_mt"),
                "flag: min median counts": g["flag"]["min_median_counts"],
                "flag: min share kept": g["flag"]["min_share_kept"],
                "reason": g["reason"],
            }
            for name, g in thresholds.items()
        ]
    ).set_index("group")
)
print(f"MIN_GENES_FOR_QC = {MIN_GENES_FOR_QC}: QC, and every view, needs at least this many genes.")

MIN_GENES_FOR_QC = 50: QC, and every view, needs at least this many genes.

show_call(
    apply_qc,
    "Leaves out control probes, computes counts, genes and the mitochondrial share per observation, keeps "
    "the observations that pass their technology's filters, and describes each dataset.",
)
qcs = []
for fam in list(adatas):
    adatas[fam], q = apply_qc(adatas[fam], thresholds)
    qcs.append(q.assign(family=fam))
qc = pd.concat(qcs, ignore_index=True)
show_table(qc)

MAX_VIOLIN_DATASETS = 40
for fam, adata in adatas.items():
    obs = adata.obs[["dataset_uid", "total_counts", "n_genes"]].copy()
    sizes = obs["dataset_uid"].value_counts()
    if len(sizes) > MAX_VIOLIN_DATASETS:
        print(f"{fam}: {len(sizes)} datasets, so the {MAX_VIOLIN_DATASETS} largest are shown.")
        obs = obs[obs["dataset_uid"].isin(sizes.index[:MAX_VIOLIN_DATASETS])]
    obs["dataset_uid"] = obs["dataset_uid"].astype(str).astype("category")
    display(Markdown(f"Counts and genes per observation after QC, family `{fam}`."))
    sc.pl.violin(ad.AnnData(obs=obs), ["total_counts", "n_genes"], groupby="dataset_uid", rotation=90, stripplot=False)

spot: 53 datasets, so the 40 largest are shown.

show_call(load_markers, "Reads the marker sets from markers.yaml, each with its citation and fallbacks.")
marker_set = load_markers()[spec["marker_set"]]
print(f"Marker set {spec['marker_set']!r}, cited from {', '.join(marker_set['citation'])}.")
show_table(
    pd.DataFrame({"markers": {c: ", ".join(g) for c, g in marker_set["compartments"].items()}}).rename_axis(
        "compartment"
    )
)
kept_after_qc = set(qc.loc[qc["status"] == "kept", "dataset_uid"].astype(str))
technology_of = cohort.set_index("uid")["technology"].astype(str)
panel_of = {
    uid: panel_id(technology_of[uid], genes)
    for adata in adatas.values()
    for uid, genes in adata.uns.get("member_genes", {}).items()
    if uid in kept_after_qc
}
used = [
    {
        "panel": pid,
        "compartment": comp,
        "genes": "not measured" if fb.get("not_measured") else ", ".join(fb.get("genes") or []),
        "note": fb.get("note", ""),
    }
    for pid, entries in (marker_set.get("fallback") or {}).items()
    if pid in set(panel_of.values())
    for comp, fb in entries.items()
]
if used:
    show_table(pd.DataFrame(used))
else:
    print("No fallback of this marker set applies to the panels of this cohort.")

Marker set 'kidney', cited from 10.1038/s41586-023-05769-3, 10.1126/science.aar2131.

show_call(with_qc_status, "Marks the members that have no cells left after QC as dropped.")
kept_members = with_qc_status(cohort, qc).query("status == 'kept'")
ENSEMBL = r"ENS[A-Z]*G\d{6}"
SYMBOL_COLUMNS = ("symbol", "gene_symbols", "gene_symbol", "gene_name", "feature_name")


def colour_gene(table, panel):
    """(var name, symbol) of the first gene of the first compartment the table measures.

    A compartment's genes are its markers, then the fallback genes for this section's own panel.
    """
    names = pd.Index(table.var_names.astype(str))
    symbols = names
    if names.str.match(ENSEMBL).mean() >= 0.5:
        column = next((c for c in SYMBOL_COLUMNS if c in table.var), None)
        if column is None:
            return None, None
        symbols = pd.Index(table.var[column].astype(str))
    by_symbol = {s.upper(): (n, s) for n, s in zip(names, symbols, strict=True)}
    fallback = (marker_set.get("fallback") or {}).get(panel) or {}
    for comp, genes in marker_set["compartments"].items():
        for gene in [*genes, *((fallback.get(comp) or {}).get("genes") or [])]:
            if gene.upper() in by_symbol:
                return by_symbol[gene.upper()]
    return None, None


def table_region(zarr_url, table_name):
    """The shapes element a table annotates, read from the table's zarr attributes without loading it."""
    group = zarr.open_group(zarr_url, mode="r", storage_options=storage_options(zarr_url) or None)
    attrs = dict(group[f"tables/{table_name}"].attrs)
    if "region" not in attrs and f"tables/{table_name}/uns/spatialdata_attrs" in group:
        attrs = ad.io.read_elem(group[f"tables/{table_name}/uns/spatialdata_attrs"])
    region = attrs.get("region")
    return region if isinstance(region, str) or region is None else (list(region) or [None])[0]


def representative_uids(members):
    """The ``representatives`` kept sections closest to the median number of observations."""
    n_obs = members.set_index("uid")["n_obs"].astype(float)
    return list((n_obs - n_obs.median()).abs().sort_values(kind="stable").index[:representatives])

for tech, members in kept_members.groupby("technology"):
    for uid in representative_uids(members):
        ds = catalog.dataset(uid)
        stored = ds.elements()
        table_name = hd_table(stored) if tech == "VisiumHD" else (getattr(ds, "default_table", None) or "table")
        region = table_region(ds.zarr_url, table_name)
        if region is None or f"shapes/{region}" not in stored:
            display(Markdown(f"{tech}, section `{uid}`: the table annotates no shapes element, so no plot."))
            continue
        # a small tissue image when there is one: lowres, else hires (never the full-resolution image)
        images = [k.split("/", 1)[1] for k in stored if k.startswith("images/")]
        lowres = next((k for k in images if "lowres" in k), None) or next((k for k in images if "hires" in k), None)
        wanted = [f"tables/{table_name}", f"shapes/{region}"] + ([f"images/{lowres}"] if lowres else [])
        sdata = ds.to_sdata(elements=wanted)
        gene, symbol = colour_gene(sdata[table_name], panel_of.get(uid))
        section = members.set_index("uid").loc[uid, "title"]
        title = f"{tech}, {section}: {symbol}" if gene else f"{tech}, {section}"
        display(Markdown(f"{tech}, section `{uid}` ({int(members.set_index('uid').loc[uid, 'n_obs'])} observations)."))
        systems = list(get_transformation(sdata.shapes[region], get_all=True))
        system = "global" if "global" in systems else systems[0]
        # spatialdata-plot 0.4.2 rasterises shapes with datashader in "global" only; elsewhere use matplotlib
        method = None if system == "global" else "matplotlib"
        # fmt: off
        if lowres:
            (
                sdata
                .pl.render_images(lowres)
                .pl.render_shapes(region, color=gene, table_name=table_name, method=method, colorbar_params={"label": symbol})
                .pl.show(
                    coordinate_systems=system,
                    title=title,
                )
            )
        else:
            (
                sdata
                .pl.render_shapes(region, color=gene, table_name=table_name, method=method, colorbar_params={"label": symbol})
                .pl.show(
                    coordinate_systems=system,
                    title=title,
                )
            )
        # fmt: on

2026-11-04 04:41:13 | [INFO] root_attr: version

2026-11-04 04:41:13 | [INFO] root_attr: multiscales

2026-11-04 04:41:13 | [INFO] root_attr: omero

2026-11-04 04:41:13 | [INFO] datasets [{'path': 's0', 'coordinateTransformations': [{'type': 'scale', 'scale': [1.0, 1.0, 1.0]}, {'type': 'translation', 'translation': [0.0, 0.0, 0.0]}]}]

2026-11-04 04:41:13 | [INFO] resolution: s0

2026-11-04 04:41:13 | [INFO]  - shape ('c', 'y', 'x') = (3, 581, 600)

2026-11-04 04:41:13 | [INFO]  - chunks =  ['3', '581', '600']

2026-11-04 04:41:13 | [INFO]  - dtype = uint8

2026-11-04 04:41:13 | [INFO] root_attr: version

2026-11-04 04:41:13 | [INFO] root_attr: multiscales

show_call(
    qc_floors, "Gives each technology's count floor, the min_counts of its QC group, for gate 3 and the joint view."
)
floors = qc_floors(thresholds)
show_table(
    pd.DataFrame(
        {
            "value": [str(x) for x in (MIN_SHARED_GENES, MAX_FLOOR_LOSS, N_HVG, N_PCS, N_NEIGHBORS, SEED)],
            "meaning": [
                "genes a view needs before a family is split per technology, and the joint view's gate 1",
                "largest share of a technology's observations that may fall below its floor on the shared genes (gate 3)",
                "highly variable genes kept when a view has more genes",
                "principal components",
                "neighbours per observation",
                "random seed",
            ],
        },
        index=["MIN_SHARED_GENES", "MAX_FLOOR_LOSS", "N_HVG", "N_PCS", "N_NEIGHBORS", "SEED"],
    )
)
show_table(pd.Series(floors, name="count floor").rename_axis("technology").to_frame())

show_call(
    embed_views,
    "Builds one view per family (or per technology when a family shares too few genes), plus the joint view "
    "when the gates pass, and embeds each with PCA, UMAP and Harmony on the section.",
)
views = embed_views(adatas, marker_set, floor=floors, use_gpu=use_gpu)
for name, v in views.items():
    p = v.uns["primer"]
    print(f"{name}: {p['n_obs']} observations, {p['n_sections']} sections; {p['genes']['rule']}. {p['note']}".strip())
first = next(iter(views.values())).uns["primer"]
for uid, reason in first["members_dropped_panel"].items():
    print(f"member {uid} {reason}")
for c in {tuple(c["uids"]): c for v in views.values() for c in v.uns["primer"]["panel_costs"]}.values():
    print(f"{c['technology']} panel of {', '.join(c['uids'])}: {c['reason']} shared genes")
for name, reason in first["views_skipped"].items():
    print(f"no view {name}: {reason}")
gates = next((v.uns["primer"]["gates"] for v in views.values() if "gates" in v.uns["primer"]), None)
if gates is not None:
    print(f"Joint view: {'yes' if gates['joint'] else 'no'}. {gates['reason']}".strip())
    for drop in gates["drops"]:
        print(f"  left out {drop['technology']} ({drop['step']}): {drop['reason']}")

spot: 364537 observations, 53 sections; 2000 highly variable genes of the 16810 measured, expressed genes (seurat_v3 on raw counts, section as batch); its 6 gene panels have 19,059 to 33,696 genes each and share 16,812.
imaging_cell: 1489193 observations, 6 sections; 2000 highly variable genes of the 5106 measured, expressed genes (seurat_v3 on raw counts, section as batch).
joint: 1795558 observations, 59 sections; 2000 highly variable genes of the 4840 measured, expressed genes (seurat_v3 on raw counts, section as batch).
Joint view: yes.

for name, v in views.items():
    colours = [c for c in ("paper", "technology", "tissue") if c in v.obs and v.obs[c].nunique() > 1]
    display(Markdown(f"View `{name}`, before Harmony."))
    sc.pl.umap(v, color=colours or None, ncols=1)
    if "X_umap_harmony" in v.obsm:
        display(Markdown(f"View `{name}`, after Harmony on the section."))
        sc.pl.embedding(v, basis="X_umap_harmony", color=colours or None, ncols=1)
    else:
        display(Markdown(f"View `{name}` has one section, so it has no Harmony embedding."))

show_call(
    mixing_by_view,
    "Measures how evenly the batches mix among each observation's nearest neighbours, per view, before and "
    "after Harmony; the bundle's manifest records the same numbers.",
)
mixing = mixing_by_view(views)
rows, definitions = [], {}
for name, by_batch in mixing.items():
    for label, by_basis in by_batch.items():
        for basis, m in by_basis.items():
            for stat in ("entropy", "other_share"):
                definitions.setdefault(stat, m[stat]["definition"].replace(m["batch_key"], "batch"))
                rows.append(
                    {
                        "view": name,
                        "batch": label,
                        "embedding": "after Harmony" if basis == "X_pca_harmony" else "before Harmony",
                        "statistic": stat,
                        "mean": m[stat]["mean"],
                        "random reference": m[stat]["random"],
                        "limit from proportions": m[stat]["random_limit"],
                        "note": m["note"],
                    }
                )
show_table(pd.DataFrame(rows))
for stat, text in definitions.items():
    print(f"{stat}: {text}")
print(CAVEAT)

entropy: How evenly the batch labels are spread among each observation's 30 nearest neighbours, divided by log(min(30, number of labels)): 0 when all neighbours share one label, 1 when the labels are as evenly spread as the neighbours allow.
other_share: The share of each observation's 30 nearest neighbours that carry another batch.
Well mixed is not the same as biologically right: a correction can also mix cells that differ for a real reason, such as tissue or disease.

fallback_genes = {}
for entries in (marker_set.get("fallback") or {}).values():
    for comp, fb in (entries or {}).items():
        fallback_genes.setdefault(comp, set()).update(str(g).upper() for g in (fb or {}).get("genes") or [])
compartment_genes = {
    comp: {str(g).upper() for g in genes} | fallback_genes.get(comp, set())
    for comp, genes in marker_set["compartments"].items()
}
readings = {}
for name, v in views.items():
    key = "nn_harmony" if "nn_harmony_distances" in v.obsp else "nn_pca"
    leiden = {"resolution": 0.5, "random_state": SEED, "neighbors_key": key, "key_added": "leiden"}
    if use_gpu:  # rapids-singlecell's n_iterations counts aggregation levels, so its default is kept
        rsc.tl.leiden(v, **leiden)
    else:
        sc.tl.leiden(v, flavor="igraph", n_iterations=2, **leiden)
    expr = v.obsm["marker_expr"]
    m = ad.AnnData(
        X=expr.to_numpy(dtype=np.float32),
        obs=v.obs[["leiden"]].copy(),
        var=pd.DataFrame(index=expr.columns.astype(str)),
    )
    display(Markdown(f"View `{name}`: {m.obs['leiden'].nunique()} clusters on `{key}`, {m.n_vars} marker genes."))
    if m.n_vars == 0 or m.obs["leiden"].nunique() < 2:
        display(Markdown("This view has fewer than two clusters or no marker gene, so there is no dotplot."))
        continue
    by_compartment = {
        comp: [g for g in expr.columns if g.upper() in genes] for comp, genes in compartment_genes.items()
    }
    sc.pl.dotplot(m, {k: g for k, g in by_compartment.items() if g}, groupby="leiden", standard_scale="var")
    scores = (
        pd.DataFrame(
            {
                comp: expr[[g for g in expr.columns if g.upper() in genes]].mean(axis=1)
                for comp, genes in compartment_genes.items()
                if any(g.upper() in genes for g in expr.columns)
            }
        )
        .groupby(m.obs["leiden"].to_numpy())
        .mean()
    )
    ranked = scores.apply(lambda r: r.sort_values(ascending=False), axis=1)
    table = pd.DataFrame(
        {
            "observations": m.obs["leiden"].value_counts().reindex(scores.index),
            "top compartment": scores.idxmax(axis=1),
            "top mean": scores.max(axis=1),
            "runner-up": [r.dropna().index[1] if r.notna().sum() > 1 else "" for _, r in ranked.iterrows()],
            "runner-up mean": [r.dropna().iloc[1] if r.notna().sum() > 1 else np.nan for _, r in ranked.iterrows()],
        }
    ).rename_axis("cluster")
    readings[name] = table
    show_table(table)

for name, table in readings.items():
    lines = [f"View `{name}`:"]
    for cluster, r in table.iterrows():
        text = (
            f"- Cluster {cluster} ({int(r['observations'])} observations) has its highest mean marker expression "
            f"for {r['top compartment']} ({r['top mean']:.2f})"
        )
        if r["runner-up"]:
            text += f", followed by {r['runner-up']} ({r['runner-up mean']:.2f})"
        lines.append(text + ".")
    display(Markdown("\n".join(lines)))

show_call(
    export_bundle,
    "Writes the manifest, umap.parquet, qc.parquet, cohort.parquet and embedding.h5ad for this cohort. A cohort whose embedding is over 1 GiB gets parts of whole datasets instead (embedding-1.h5ad, embedding-2.h5ad, ...), and anndata.concat over the parts gives the whole cohort.",
)
bundle = export_bundle(
    views,
    cohort,
    qc,
    spec,
    out_dir,
    release,
    runtime_s=time.time() - start,
    mixing=mixing,
    primers_ref=primers_ref,
)
for f in sorted(bundle.iterdir()):
    print(f"{f.relative_to(out_dir)}  {f.stat().st_size} bytes")
manifest = json.loads((bundle / "manifest.json").read_text())
for key in (
    "slug",
    "release",
    "primers_ref",
    "n_datasets",
    "n_papers",
    "technologies",
    "n_cells_before",
    "n_cells_after",
    "n_flagged",
    "path",
    "hardware",
    "runtime_s",
):
    print(f"{key}: {manifest[key]}")

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

command = (
    f"papermill primers/notebooks/primer.ipynb primer.ipynb -p slug {slug} -p release {release} "
    f"-p primers_ref {primers_ref} -p use_gpu {use_gpu}" + (f" -p smoke {smoke}" if smoke else "")
)
versions = "\n".join(f"  - {k} {v}" for k, v in manifest["versions"].items())
display(
    Markdown(
        f"- Catalog release: `{release}`\n- primers reference: `{primers_ref}`\n- Versions:\n{versions}\n"
        f"- Hardware: {manifest['hardware']} ({platform.system()})\n- Path: {manifest['path']}\n\n"
        "To run this primer again:"
    )
)
show_command(command)

papermill primers/notebooks/primer.ipynb primer.ipynb -p slug mouse-kidney -p release 2026-11-01 -p primers_ref 4d7af340849c00077f9d21c5b54af9e689a4a070 -p use_gpu True

	uid	title	technology	family	paper	paper_source	licence	n_obs	status
0	S8nURiTjtQ5opsAx0000	visium-hd-three-prime-mouse-kidney	VisiumHD	spot	10x Genomics demo datasets	10x_demo	CC-BY-4.0	423482	kept
1	1uZYdlOkO6lJ01us0000	mouse-tissue-microarray-in-5x5-layout-with-1-mm-edge-to-edge-spacing-ffpe-2-standard	Visium	spot	10x Genomics demo datasets	10x_demo	CC-BY-4.0	2805	kept
2	SBUE1JyDEWoFeqL00000	mouse-kidney-section-coronal-1-standard-1-1-0	Visium	spot	10x Genomics demo datasets	10x_demo	CC-BY-4.0	1438	kept
3	OdXk8AQvash9eXa00000	mouse-kidney-section-coronal-1-standard-1-0-0	Visium	spot	10x Genomics demo datasets	10x_demo	CC-BY-4.0	1438	dropped: 10x re-release of a section kept in a newer release
4	9z8u3Dix4eGkso0N0000	GSM8659289	Visium	spot	geo-gse283283	geo_series	ncbi-geo-policy	4009	kept
...	...	...	...	...	...	...	...	...	...
59	skpT5oSQUZpQnTHJ0000	GSM8659234	Visium	spot	geo-gse283283	geo_series	ncbi-geo-policy	3822	kept
60	kFkwJSOM5HovKIGn0000	GSM8659310	Visium	spot	geo-gse283283	geo_series	ncbi-geo-policy	3391	kept
61	EnQKFrJwldFUBmIt0000	GSM8659165	Visium	spot	geo-gse283283	geo_series	ncbi-geo-policy	3219	kept
62	QD7nbkMF1FaeC0vD0000	GSM8323121	Visium	spot	geo-gse269622	geo_series	ncbi-geo-policy	3091	kept
63	hY6i807e0B4navqX0000	GSM8659229	Visium	spot	geo-gse283283	geo_series	ncbi-geo-policy	3306	kept

	dataset_uid	technology	n_before	n_after	median_counts	median_genes	pct_mt_median	flagged	flag_reason	status	family
0	1C4Vb3fEtnXsDroX0000	Visium	3124	3124	30379.0	6956.0	0.0	False		kept	spot
1	1EWiyjxa6QPuxqh20000	Visium	1835	1832	34439.0	5782.0	13.067	False		kept	spot
2	1uZYdlOkO6lJ01us0000	Visium	2805	2798	66412.0	9207.0	0.0	False		kept	spot
3	2BjfNGIrAIYZkuqa0000	Visium	1598	1374	10773.0	3515.5	14.392	False		kept	spot
4	4h5ljZYfb1MnBw1w0000	Visium	2051	2050	39328.0	8430.0	0.0	False		kept	spot
5	5myRE3jozziJrraD0000	Visium	2116	2113	16432.5	5387.0	4.681	False		kept	spot
6	5xpnmM5MmdlrVrJb0000	Visium	2219	1665	3345.0	1322.0	23.644	False		kept	spot
7	76q9yU2ThGJHrgDt0000	Visium	1617	1613	8010.0	2729.0	13.191	False		kept	spot
8	77Wqbbj0goz4LT2n0000	Visium	1893	1856	8018.0	2806.0	13.658	False		kept	spot
9	9NNCVo7S6i8m6vNj0000	Visium	3221	3139	10328.0	4097.0	5.176	False		kept	spot
10	9z8u3Dix4eGkso0N0000	Visium	4009	3710	3056.0	1665.0	8.984	False		kept	spot
11	AMW0iIXSwqyUZcPw0000	Visium	3915	3912	18168.0	5392.0	0.0	False		kept	spot
12	AuvzJcuF17sC6sSw0000	Visium	3453	3376	4880.0	2109.0	8.596	False		kept	spot
13	EnQKFrJwldFUBmIt0000	Visium	3219	3099	14250.0	4765.0	6.865	False		kept	spot
14	EsKRKjncF0VQOrHD0000	Visium	3502	3497	10016.0	3291.5	13.779	False		kept	spot
15	F0wrvAY9gjv0VGrL0000	Visium	2027	2025	37350.0	8141.0	0.0	False		kept	spot
16	Fiahq4OKIYvpQqXt0000	Visium	3772	2941	2376.0	1422.5	2.997	False		kept	spot
17	G5gKK4HV2I9lxhzk0000	Visium	2829	2827	10767.0	4216.0	2.772	False		kept	spot
18	GnuLSu2xWXM4pTC30000	Visium	2772	2754	23133.5	7845.0	0.0	False		kept	spot
19	HVKVnPeTrMFTx9Ty0000	VisiumHD	77975	76660	3079.0	2146.0	1.751	False		kept	spot
20	LK6ojhhlUQ2kjXjq0000	Visium	2300	2299	34196.5	7521.5	0.0	False		kept	spot
21	Q4WOm49uPIu3NQ6O0000	Visium	2320	2320	52235.0	9083.0	0.0	False		kept	spot
22	QD7nbkMF1FaeC0vD0000	Visium	3091	3085	19936.0	5469.0	0.0	False		kept	spot
23	RiHSH9qnU5DTQbDq0000	Visium	2151	2150	38475.0	7834.0	0.0	False		kept	spot
24	S8nURiTjtQ5opsAx0000	VisiumHD	106302	100316	1240.0	776.0	9.655	False		kept	spot
25	SBUE1JyDEWoFeqL00000	Visium	1438	1433	38308.0	6516.5	11.733	False		kept	spot
26	T20h7X1RZrsEg8O80000	VisiumHD	60302	58751	3163.5	2277.0	2.053	False		kept	spot
27	TaN3zCkXmVMXuKEv0000	Visium	3754	3702	7680.5	2772.5	14.487	False		kept	spot
28	Tdf1vsWTZRjxhylV0000	Visium	3489	3462	4896.0	2043.0	16.766	False		kept	spot
29	Ud1hNpoaire2AHXv0000	Visium	3888	3886	15002.0	4919.0	4.841	False		kept	spot
30	V4BSuEVSwZCfoPoI0000	Visium	3254	3243	5935.0	2600.5	7.477	False		kept	spot
31	VTH4sWHFN0KuPjrV0000	Visium	1058	642	13777.0	3458.0	28.314	False		kept	spot
32	VVvS0QC3Hcch6qaD0000	Visium	1815	1794	6525.0	2367.0	14.161	False		kept	spot
33	WZSaovLCsJkiWF2c0000	Visium	1888	1884	31796.0	5730.5	11.366	False		kept	spot
34	WgSIoOfGcSFdRJ4H0000	Visium	3615	3570	2557.0	1229.0	19.983	False		kept	spot
35	Wgc4scrg3GorWqMt0000	Visium	1119	201	12478.0	3067.0	35.624	True	18% kept < 50%	kept	spot
36	XbE3kPoLb7oafNHX0000	Visium	1306	671	13370.5	3393.0	29.533	False		kept	spot
37	ZqvUi2qMUmCqEOVJ0000	Visium	2423	2420	54521.0	9172.0	0.0	False		kept	spot
38	agDrf8Dv9VyqA0si0000	Visium	2064	2053	28992.5	5647.0	12.652	False		kept	spot
39	cVoMz8bw6TION1TQ0000	Visium	2218	2218	24628.5	5487.5	0.0	False		kept	spot
40	eBKZeXpdSQs59E600000	Visium	3299	3295	8338.0	3434.0	5.096	False		kept	spot
41	eHY6Ce9SwGfZ0Kc50000	Visium	3393	3392	15081.0	5300.0	0.0	False		kept	spot
42	gRmSJXC079KyfxwM0000	Visium	2858	2855	11928.5	5298.5	0.0	False		kept	spot
43	gh8oCMDt2w5aAksS0000	Visium	3443	3434	21052.0	6307.0	0.0	False		kept	spot
44	hY6i807e0B4navqX0000	Visium	3306	3284	10214.0	3572.5	8.871	False		kept	spot
45	hqxIa94cm5zFiA2n0000	Visium	2212	2199	10789.0	3563.0	7.841	False		kept	spot
46	hvM03A2y9tXEL4Zu0000	Visium	2976	2954	12183.0	4113.0	5.657	False		kept	spot
47	i2TlgWlpuW5Mqaga0000	Visium	1094	688	15494.5	3785.5	27.447	False		kept	spot
48	jemKplLVhobfVxbn0000	Visium	3722	3696	8126.5	3126.5	9.118	False		kept	spot
49	kFkwJSOM5HovKIGn0000	Visium	3391	3389	9021.0	3694.0	4.224	False		kept	spot
50	s2bGD9UQmHk6Ieev0000	Visium	1974	1974	46470.5	9076.5	0.0	False		kept	spot
51	skpT5oSQUZpQnTHJ0000	Visium	3822	3758	3161.0	1468.0	20.16	False		kept	spot
52	u4BR1Qk37AR6WsHq0000	Visium	3328	3154	5260.0	2529.0	3.55	False		kept	spot
53	3ba4J0QZF4uXE02L0000	Xenium	244107	243709	1131.0	714.0	n/a	False		kept	imaging_cell
54	CFj4xLRXNr43gBnR0000	Xenium	304788	304361	1022.0	674.0	n/a	False		kept	imaging_cell
55	UWwYUTqhCqtRp4ow0000	Xenium	111807	111605	854.0	589.0	n/a	False		kept	imaging_cell
56	eXYyyHrwdx5L9iL80000	Xenium	328589	328210	970.0	647.0	n/a	False		kept	imaging_cell
57	g0CnsMo9BuZhYzR00000	Xenium	172225	171936	1222.0	761.0	n/a	False		kept	imaging_cell
58	ssfAnmTU7tHlRb7l0000	Xenium	329957	329372	903.0	616.0	n/a	False		kept	imaging_cell

	markers
compartment
proximal_tubule	LRP2, CUBN, SLC34A1
thick_ascending_limb	UMOD, SLC12A1
distal_tubule	SLC12A3, CALB1
collecting_duct_principal	AQP2, AQP3
collecting_duct_intercalated	ATP6V1B1, SLC4A1
podocyte	NPHS1, NPHS2, WT1
endothelial	PECAM1, EMCN, PLVAP
fibroblast	DCN, COL1A1, PDGFRB
immune	PTPRC, CD68, CD3E

	count floor
technology
Visium	500.0
SpatialTranscriptomics	500.0
VisiumHD	200.0
Xenium	10.0
Merscope	10.0
MERFISH	10.0

	observations	top compartment	top mean	runner-up	runner-up mean
cluster
0	33157	fibroblast	1.542	collecting_duct_principal	0.062
1	5860	podocyte	1.21	collecting_duct_principal	0.067
10	26597	proximal_tubule	1.03	collecting_duct_principal	0.391
11	29196	fibroblast	0.614	collecting_duct_principal	0.068
12	8705	fibroblast	0.611	collecting_duct_principal	0.546
13	10949	proximal_tubule	1.376	collecting_duct_principal	0.057
14	977	fibroblast	1.167	collecting_duct_principal	0.01
2	5292	fibroblast	0.508	collecting_duct_principal	0.179
3	14086	distal_tubule	0.954	collecting_duct_principal	0.751
4	42686	thick_ascending_limb	1.321	collecting_duct_principal	0.657
5	49287	fibroblast	1.154	collecting_duct_principal	0.095
6	5458	fibroblast	0.714	collecting_duct_principal	0.334
7	7858	proximal_tubule	1.002	collecting_duct_principal	0.159
8	56315	thick_ascending_limb	0.873	collecting_duct_principal	0.242
9	68114	proximal_tubule	1.354	collecting_duct_principal	0.216

Cohort primer¶

Mouse kidney¶

Setup¶

Query and cohort¶

Quality control¶

Marker set¶

Spatial overview¶

Embedding¶

Biology¶

Draft reading, to be reviewed by the Lead¶

Export and reproduce¶

	status	members
0	kept	59
1	dropped: no paper (no study_id, series or collection)	4
2	dropped: 10x re-release of a section kept in a newer release	1

	technologies	min_counts	min_genes	max_pct_mt	flag: min median counts	flag: min share kept	reason
group
spot	Visium, SpatialTranscriptomics	500	250	30.0	1000	0.5	A Visium or ST spot holds about 1 to 10 cells, so a spot with under 500 UMIs or 250 genes is mostly background or tissue edge; solid tissues often reach 20 to 30% mitochondrial reads, so the cap is 30%.
binned_spot	VisiumHD	200	50	30.0	200	0.5	On the 5 Visium HD sections of oliveira-2024-colorectal, 16 um bins at 200 counts kept 74% of bins with the same marker signal as 8 um bins at Oliveira et al.'s 100 (Nat Genet 57:1512-1523), which kept 60%, and found T cells in all 5 sections against 3 (web tasks/2026-11-01-t094-hd-bin-size.md).
imaging_cell	Xenium, Merscope, MERFISH	10	3	n/a	30	0.5	Targeted panels give tens to hundreds of decoded transcripts per segmented cell, and a cell with under 10 transcripts or 3 genes cannot be given a type; the panels have no mitochondrial genes.

	value	meaning
MIN_SHARED_GENES	200	genes a view needs before a family is split per technology, and the joint view's gate 1
MAX_FLOOR_LOSS	0.5	largest share of a technology's observations that may fall below its floor on the shared genes (gate 3)
N_HVG	2000	highly variable genes kept when a view has more genes
N_PCS	30	principal components
N_NEIGHBORS	30	neighbours per observation
SEED	0	random seed

	view	batch	embedding	statistic	mean	random reference	limit from proportions	note
0	spot	section	before Harmony	entropy	0.109	0.614	0.794
1	spot	section	before Harmony	other_share	0.197	0.823	0.851
2	spot	section	after Harmony	entropy	0.33	0.613	0.794
3	spot	section	after Harmony	other_share	0.483	0.823	0.851
4	spot	paper	before Harmony	entropy	0.015	0.624	0.685
5	spot	paper	before Harmony	other_share	0.018	0.713	0.738
6	spot	paper	after Harmony	entropy	0.221	0.624	0.685
7	spot	paper	after Harmony	other_share	0.246	0.713	0.738
8	imaging_cell	section	before Harmony	entropy	0.799	0.916	0.966
9	imaging_cell	section	before Harmony	other_share	0.694	0.788	0.815
10	imaging_cell	section	after Harmony	entropy	0.875	0.916	0.966
11	imaging_cell	section	after Harmony	other_share	0.758	0.788	0.815
12	imaging_cell	paper	before Harmony	entropy	n/a	n/a	n/a	one paper, so neither statistic is defined
13	imaging_cell	paper	before Harmony	other_share	n/a	n/a	n/a	one paper, so neither statistic is defined
14	imaging_cell	paper	after Harmony	entropy	n/a	n/a	n/a	one paper, so neither statistic is defined
15	imaging_cell	paper	after Harmony	other_share	n/a	n/a	n/a	one paper, so neither statistic is defined
16	joint	section	before Harmony	entropy	0.387	0.592	0.697
17	joint	section	before Harmony	other_share	0.638	0.84	0.869
18	joint	section	after Harmony	entropy	0.467	0.592	0.697
19	joint	section	after Harmony	other_share	0.738	0.84	0.869
20	joint	paper	before Harmony	entropy	0.005	0.258	0.304
21	joint	paper	before Harmony	other_share	0.005	0.295	0.304
22	joint	paper	after Harmony	entropy	0.066	0.258	0.304
23	joint	paper	after Harmony	other_share	0.083	0.295	0.304
24	joint	technology	before Harmony	entropy	0.0	0.486	0.519
25	joint	technology	before Harmony	other_share	0.0	0.287	0.297
26	joint	technology	after Harmony	entropy	0.068	0.486	0.519
27	joint	technology	after Harmony	other_share	0.052	0.287	0.297

	observations	top compartment	top mean	runner-up	runner-up mean
cluster
0	19600	fibroblast	0.423	collecting_duct_principal	0.012
1	50109	fibroblast	0.24	collecting_duct_principal	0.014
10	25918	fibroblast	1.482	collecting_duct_principal	0.021
11	203314	fibroblast	0.516	collecting_duct_principal	0.005
2	159309	fibroblast	0.689	collecting_duct_principal	0.017
3	163234	distal_tubule	0.477	collecting_duct_principal	0.149
4	135720	endothelial	0.667	collecting_duct_principal	0.01
5	126314	collecting_duct_principal	0.301	collecting_duct_principal	0.301
6	220046	podocyte	0.375	collecting_duct_principal	0.006
7	89485	proximal_tubule	0.49	collecting_duct_principal	0.011
8	180389	podocyte	0.314	collecting_duct_principal	0.004
9	115755	podocyte	0.504	collecting_duct_principal	0.008

	observations	top compartment	top mean	runner-up	runner-up mean
cluster
0	18479	fibroblast	0.555	collecting_duct_principal	0.027
1	212584	fibroblast	0.776	collecting_duct_principal	0.019
10	161070	proximal_tubule	0.899	collecting_duct_principal	0.219
11	41190	fibroblast	1.721	collecting_duct_principal	0.014
12	185145	fibroblast	0.495	collecting_duct_principal	0.005
13	6706	fibroblast	0.463	collecting_duct_principal	0.018
14	212	thick_ascending_limb	1.196	collecting_duct_principal	0.496
2	166985	endothelial	0.643	collecting_duct_principal	0.014
3	171006	distal_tubule	0.477	collecting_duct_principal	0.171
4	164926	podocyte	0.779	collecting_duct_principal	0.009
5	64172	fibroblast	0.342	collecting_duct_principal	0.009
6	104238	podocyte	0.207	collecting_duct_principal	0.01
7	129295	collecting_duct_principal	0.336	collecting_duct_principal	0.336
8	276522	podocyte	0.51	collecting_duct_principal	0.005
9	93028	fibroblast	0.494	collecting_duct_principal	0.46