slug = "human-colon"
release = "2026-10-28"
use_gpu = True
out_dir = "bundles"
smoke = 0
primers_ref = "feat-primers"
cohorts_path = None
catalog_path = None
representatives = 1
dev_note = ""

# Parameters
slug = "human-kidney"
release = "2026-11-01"
out_dir = "bundles"
primers_ref = "4d7af340849c00077f9d21c5b54af9e689a4a070"
dev_note = ""

pip install "sddb-primers @ git+https://github.com/theislab/spatialdata-db-client@4d7af340849c00077f9d21c5b54af9e689a4a070#subdirectory=primers"

import inspect
import json
import platform
import time

import anndata as ad
import numpy as np
import pandas as pd
import scanpy as sc
import spatialdata_plot  # noqa: F401  (registers the .pl accessor)
import zarr
from primers.cohort import hd_table, load_cohort, smoke_spec
from primers.embed import CAVEAT, MAX_FLOOR_LOSS, MIN_SHARED_GENES, N_HVG, N_NEIGHBORS, N_PCS, SEED, embed_views
from primers.export import export_bundle, mixing_by_view, with_qc_status
from primers.markers import load_markers, panel_id
from primers.qc import MIN_GENES_FOR_QC, apply_qc, load_thresholds, qc_floors
from spatialdata.transformations import get_transformation

import sddb
from sddb.remote import storage_options

start = time.time()
out_dir = Path(out_dir)
if use_gpu:
    import cupy as cp
    import rapids_singlecell as rsc
    import rmm
    from rmm.allocators.cupy import rmm_cupy_allocator

    rmm.reinitialize(managed_memory=True)
    cp.cuda.set_allocator(rmm_cupy_allocator)


def show_call(func, what):
    """Show what a primers function does and link its source at ``primers_ref``."""
    _, line = inspect.getsourcelines(func)
    module = func.__module__.split(".")[-1]
    url = f"{REPO}/blob/{primers_ref}/primers/primers/{module}.py#L{line}"
    display(Markdown(f"`{func.__module__}.{func.__name__}`: {what} [Source at `{primers_ref}`]({url})."))


def show_table(df, digits=3):
    """Display ``df`` rounded, with missing values written as n/a, and long text columns in full."""
    shown = df.round(digits).astype(object)
    with pd.option_context("display.max_colwidth", None):
        display(shown.where(df.notna(), "n/a"))


catalog = sddb.Catalog.from_file(catalog_path) if catalog_path else sddb.Catalog(version=release)

if smoke:
    show_call(
        smoke_spec,
        "Keeps only the smallest kept member of each of a few papers, taken in turn from each family, for a "
        "quick end-to-end check.",
    )
    n_spec_excluded = len(spec.get("exclude_uids") or [])
    spec = smoke_spec(spec, catalog, smoke)
    print(
        f"Smoke run on {smoke} papers: {len(spec['exclude_uids']) - n_spec_excluded} more uids are excluded "
        f"({len(spec['exclude_uids'])} in all)."
    )
for key in ("slug", "query", "include_uids", "exclude_uids", "families", "marker_set", "rationale"):
    value = spec.get(key)
    if smoke and key == "exclude_uids":
        value = f"{len(value)} uids"
    print(f"{key}: {value}")

slug: human-kidney
query: {'organism': 'human', 'tissue': 'kidney', 'n_obs__lte': 250000}
include_uids: None
exclude_uids: ['QhWLkUPIwgReWxUl0000', '7cvm04SitwDiUccp0000', 'GrL6vmD9cQIX9kWy0000', '0hvcEnilQM3UbBEX0000', 'TPUEOPBx6Bl9XSwV0000', 'RxMxBceGIVY7cunl0000', '9ECHPci2ByJEXpmm0000', 'xR5R5oIEI5vC5rw40000', 'jVoLDQ0g9HrFR5090000', 'L63qBPEi1Jb14kUM0000', 'GIvMm2g196WM29FC0000', 'WUuICN3yJz9v0cHY0000', 'xtDuriWbektArUTi0000', 'sVYWy3ftQNgttMRP0000', 'H7X7fQqJO0v9XX8T0000', 'NHyd5DFysygNrkdu0000', 'kZEVJcHN2Kwfs9W70000', 'sGZxtkw1WXUZ7YFq0000']
families: ['spot', 'imaging']
marker_set: kidney
rationale: Diabetic, glomerular and cancer kidney on Visium, Visium HD and Xenium across many labs; the cap leaves out the largest Xenium and HD sections, and a 50-gene Xenium series is left out.

show_call(
    load_cohort,
    "Runs the query on the catalog, adds the included uids, removes the excluded ones, and loads each kept "
    "member's table keyed by gene symbol, joined within its technology family.",
)
adatas, cohort = load_cohort(spec, catalog=catalog)

columns = ["uid", "title", "technology", "family", "paper", "paper_source", "license", "n_obs", "status"]
show_table(cohort[columns].rename(columns={"license": "licence"}))
show_table(cohort["status"].value_counts().rename_axis("status").reset_index(name="members"))
print(f"{int((cohort['status'] == 'kept').sum())} kept, {int((cohort['status'] != 'kept').sum())} dropped.")

144 kept, 13 dropped.

show_call(load_thresholds, "Reads the QC threshold groups from thresholds.yaml.")
thresholds = load_thresholds()
show_table(
    pd.DataFrame(
        [
            {
                "group": name,
                "technologies": ", ".join(g["technologies"]),
                "min_counts": g["min_counts"],
                "min_genes": g["min_genes"],
                "max_pct_mt": g.get("max_pct_mt"),
                "flag: min median counts": g["flag"]["min_median_counts"],
                "flag: min share kept": g["flag"]["min_share_kept"],
                "reason": g["reason"],
            }
            for name, g in thresholds.items()
        ]
    ).set_index("group")
)
print(f"MIN_GENES_FOR_QC = {MIN_GENES_FOR_QC}: QC, and every view, needs at least this many genes.")

MIN_GENES_FOR_QC = 50: QC, and every view, needs at least this many genes.

show_call(
    apply_qc,
    "Leaves out control probes, computes counts, genes and the mitochondrial share per observation, keeps "
    "the observations that pass their technology's filters, and describes each dataset.",
)
qcs = []
for fam in list(adatas):
    adatas[fam], q = apply_qc(adatas[fam], thresholds)
    qcs.append(q.assign(family=fam))
qc = pd.concat(qcs, ignore_index=True)
show_table(qc)

MAX_VIOLIN_DATASETS = 40
for fam, adata in adatas.items():
    obs = adata.obs[["dataset_uid", "total_counts", "n_genes"]].copy()
    sizes = obs["dataset_uid"].value_counts()
    if len(sizes) > MAX_VIOLIN_DATASETS:
        print(f"{fam}: {len(sizes)} datasets, so the {MAX_VIOLIN_DATASETS} largest are shown.")
        obs = obs[obs["dataset_uid"].isin(sizes.index[:MAX_VIOLIN_DATASETS])]
    obs["dataset_uid"] = obs["dataset_uid"].astype(str).astype("category")
    display(Markdown(f"Counts and genes per observation after QC, family `{fam}`."))
    sc.pl.violin(ad.AnnData(obs=obs), ["total_counts", "n_genes"], groupby="dataset_uid", rotation=90, stripplot=False)

spot: 124 datasets, so the 40 largest are shown.

show_call(load_markers, "Reads the marker sets from markers.yaml, each with its citation and fallbacks.")
marker_set = load_markers()[spec["marker_set"]]
print(f"Marker set {spec['marker_set']!r}, cited from {', '.join(marker_set['citation'])}.")
show_table(
    pd.DataFrame({"markers": {c: ", ".join(g) for c, g in marker_set["compartments"].items()}}).rename_axis(
        "compartment"
    )
)
kept_after_qc = set(qc.loc[qc["status"] == "kept", "dataset_uid"].astype(str))
technology_of = cohort.set_index("uid")["technology"].astype(str)
panel_of = {
    uid: panel_id(technology_of[uid], genes)
    for adata in adatas.values()
    for uid, genes in adata.uns.get("member_genes", {}).items()
    if uid in kept_after_qc
}
used = [
    {
        "panel": pid,
        "compartment": comp,
        "genes": "not measured" if fb.get("not_measured") else ", ".join(fb.get("genes") or []),
        "note": fb.get("note", ""),
    }
    for pid, entries in (marker_set.get("fallback") or {}).items()
    if pid in set(panel_of.values())
    for comp, fb in entries.items()
]
if used:
    show_table(pd.DataFrame(used))
else:
    print("No fallback of this marker set applies to the panels of this cohort.")

Marker set 'kidney', cited from 10.1038/s41586-023-05769-3, 10.1126/science.aar2131.

show_call(with_qc_status, "Marks the members that have no cells left after QC as dropped.")
kept_members = with_qc_status(cohort, qc).query("status == 'kept'")
ENSEMBL = r"ENS[A-Z]*G\d{6}"
SYMBOL_COLUMNS = ("symbol", "gene_symbols", "gene_symbol", "gene_name", "feature_name")


def colour_gene(table, panel):
    """(var name, symbol) of the first gene of the first compartment the table measures.

    A compartment's genes are its markers, then the fallback genes for this section's own panel.
    """
    names = pd.Index(table.var_names.astype(str))
    symbols = names
    if names.str.match(ENSEMBL).mean() >= 0.5:
        column = next((c for c in SYMBOL_COLUMNS if c in table.var), None)
        if column is None:
            return None, None
        symbols = pd.Index(table.var[column].astype(str))
    by_symbol = {s.upper(): (n, s) for n, s in zip(names, symbols, strict=True)}
    fallback = (marker_set.get("fallback") or {}).get(panel) or {}
    for comp, genes in marker_set["compartments"].items():
        for gene in [*genes, *((fallback.get(comp) or {}).get("genes") or [])]:
            if gene.upper() in by_symbol:
                return by_symbol[gene.upper()]
    return None, None


def table_region(zarr_url, table_name):
    """The shapes element a table annotates, read from the table's zarr attributes without loading it."""
    group = zarr.open_group(zarr_url, mode="r", storage_options=storage_options(zarr_url) or None)
    attrs = dict(group[f"tables/{table_name}"].attrs)
    if "region" not in attrs and f"tables/{table_name}/uns/spatialdata_attrs" in group:
        attrs = ad.io.read_elem(group[f"tables/{table_name}/uns/spatialdata_attrs"])
    region = attrs.get("region")
    return region if isinstance(region, str) or region is None else (list(region) or [None])[0]


def representative_uids(members):
    """The ``representatives`` kept sections closest to the median number of observations."""
    n_obs = members.set_index("uid")["n_obs"].astype(float)
    return list((n_obs - n_obs.median()).abs().sort_values(kind="stable").index[:representatives])

for tech, members in kept_members.groupby("technology"):
    for uid in representative_uids(members):
        ds = catalog.dataset(uid)
        stored = ds.elements()
        table_name = hd_table(stored) if tech == "VisiumHD" else (getattr(ds, "default_table", None) or "table")
        region = table_region(ds.zarr_url, table_name)
        if region is None or f"shapes/{region}" not in stored:
            display(Markdown(f"{tech}, section `{uid}`: the table annotates no shapes element, so no plot."))
            continue
        # a small tissue image when there is one: lowres, else hires (never the full-resolution image)
        images = [k.split("/", 1)[1] for k in stored if k.startswith("images/")]
        lowres = next((k for k in images if "lowres" in k), None) or next((k for k in images if "hires" in k), None)
        wanted = [f"tables/{table_name}", f"shapes/{region}"] + ([f"images/{lowres}"] if lowres else [])
        sdata = ds.to_sdata(elements=wanted)
        gene, symbol = colour_gene(sdata[table_name], panel_of.get(uid))
        section = members.set_index("uid").loc[uid, "title"]
        title = f"{tech}, {section}: {symbol}" if gene else f"{tech}, {section}"
        display(Markdown(f"{tech}, section `{uid}` ({int(members.set_index('uid').loc[uid, 'n_obs'])} observations)."))
        systems = list(get_transformation(sdata.shapes[region], get_all=True))
        system = "global" if "global" in systems else systems[0]
        # spatialdata-plot 0.4.2 rasterises shapes with datashader in "global" only; elsewhere use matplotlib
        method = None if system == "global" else "matplotlib"
        # fmt: off
        if lowres:
            (
                sdata
                .pl.render_images(lowres)
                .pl.render_shapes(region, color=gene, table_name=table_name, method=method, colorbar_params={"label": symbol})
                .pl.show(
                    coordinate_systems=system,
                    title=title,
                )
            )
        else:
            (
                sdata
                .pl.render_shapes(region, color=gene, table_name=table_name, method=method, colorbar_params={"label": symbol})
                .pl.show(
                    coordinate_systems=system,
                    title=title,
                )
            )
        # fmt: on

2026-11-04 05:31:02 | [INFO] root_attr: version

2026-11-04 05:31:02 | [INFO] root_attr: multiscales

2026-11-04 05:31:02 | [INFO] root_attr: omero

2026-11-04 05:31:02 | [INFO] datasets [{'path': 's0', 'coordinateTransformations': [{'type': 'scale', 'scale': [1.0, 1.0, 1.0]}, {'type': 'translation', 'translation': [0.0, 0.0, 0.0]}]}]

2026-11-04 05:31:03 | [INFO] resolution: s0

2026-11-04 05:31:03 | [INFO]  - shape ('c', 'y', 'x') = (3, 600, 527)

2026-11-04 05:31:03 | [INFO]  - chunks =  ['3', '600', '527']

2026-11-04 05:31:03 | [INFO]  - dtype = uint8

2026-11-04 05:31:03 | [INFO] root_attr: version

2026-11-04 05:31:03 | [INFO] root_attr: multiscales

show_call(
    qc_floors, "Gives each technology's count floor, the min_counts of its QC group, for gate 3 and the joint view."
)
floors = qc_floors(thresholds)
show_table(
    pd.DataFrame(
        {
            "value": [str(x) for x in (MIN_SHARED_GENES, MAX_FLOOR_LOSS, N_HVG, N_PCS, N_NEIGHBORS, SEED)],
            "meaning": [
                "genes a view needs before a family is split per technology, and the joint view's gate 1",
                "largest share of a technology's observations that may fall below its floor on the shared genes (gate 3)",
                "highly variable genes kept when a view has more genes",
                "principal components",
                "neighbours per observation",
                "random seed",
            ],
        },
        index=["MIN_SHARED_GENES", "MAX_FLOOR_LOSS", "N_HVG", "N_PCS", "N_NEIGHBORS", "SEED"],
    )
)
show_table(pd.Series(floors, name="count floor").rename_axis("technology").to_frame())

show_call(
    embed_views,
    "Builds one view per family (or per technology when a family shares too few genes), plus the joint view "
    "when the gates pass, and embeds each with PCA, UMAP and Harmony on the section.",
)
views = embed_views(adatas, marker_set, floor=floors, use_gpu=use_gpu)
for name, v in views.items():
    p = v.uns["primer"]
    print(f"{name}: {p['n_obs']} observations, {p['n_sections']} sections; {p['genes']['rule']}. {p['note']}".strip())
first = next(iter(views.values())).uns["primer"]
for uid, reason in first["members_dropped_panel"].items():
    print(f"member {uid} {reason}")
for c in {tuple(c["uids"]): c for v in views.values() for c in v.uns["primer"]["panel_costs"]}.values():
    print(f"{c['technology']} panel of {', '.join(c['uids'])}: {c['reason']} shared genes")
for name, reason in first["views_skipped"].items():
    print(f"no view {name}: {reason}")
gates = next((v.uns["primer"]["gates"] for v in views.values() if "gates" in v.uns["primer"]), None)
if gates is not None:
    print(f"Joint view: {'yes' if gates['joint'] else 'no'}. {gates['reason']}".strip())
    for drop in gates["drops"]:
        print(f"  left out {drop['technology']} ({drop['step']}): {drop['reason']}")

spot: 276312 observations, 124 sections; 2000 highly variable genes of the 16849 measured, expressed genes (seurat_v3 on raw counts, section as batch); its 7 gene panels have 17,943 to 37,082 genes each and share 16,851.
imaging_cell: 878788 observations, 17 sections; all 129 measured, expressed genes of the view; its 3 gene panels have 377 to 5,092 genes each and share 129. only 129 shared genes (under 200)
member 4aa7meIGBU5btyG60000 dropped: panel shares 7 genes with the rest of its technology
member F5aA0qoFwU5KqSQP0000 dropped: panel shares 7 genes with the rest of its technology
member E5f35NJ4oQ4NGo2J0000 dropped: panel shares 7 genes with the rest of its technology
Xenium panel of AW1AUSyQ5HJuvJUp0000: kept: shares 129 of 224 shared genes
Xenium panel of USOaqAveOg9ScruM0001, RwEKt4wbzJsqG4yK0001, W1bZvhwcfHpd4PID0000: kept: shares 129 of 198 shared genes
Joint view: no. fewer than two families keep a technology
  left out Visium (gate3): loses 99% of its observations below 500 counts on the 124 shared genes (limit 50%)
  left out VisiumHD (gate3): loses 100% of its observations below 200 counts on the 124 shared genes (limit 50%)

for name, v in views.items():
    colours = [c for c in ("paper", "technology", "tissue") if c in v.obs and v.obs[c].nunique() > 1]
    display(Markdown(f"View `{name}`, before Harmony."))
    sc.pl.umap(v, color=colours or None, ncols=1)
    if "X_umap_harmony" in v.obsm:
        display(Markdown(f"View `{name}`, after Harmony on the section."))
        sc.pl.embedding(v, basis="X_umap_harmony", color=colours or None, ncols=1)
    else:
        display(Markdown(f"View `{name}` has one section, so it has no Harmony embedding."))

show_call(
    mixing_by_view,
    "Measures how evenly the batches mix among each observation's nearest neighbours, per view, before and "
    "after Harmony; the bundle's manifest records the same numbers.",
)
mixing = mixing_by_view(views)
rows, definitions = [], {}
for name, by_batch in mixing.items():
    for label, by_basis in by_batch.items():
        for basis, m in by_basis.items():
            for stat in ("entropy", "other_share"):
                definitions.setdefault(stat, m[stat]["definition"].replace(m["batch_key"], "batch"))
                rows.append(
                    {
                        "view": name,
                        "batch": label,
                        "embedding": "after Harmony" if basis == "X_pca_harmony" else "before Harmony",
                        "statistic": stat,
                        "mean": m[stat]["mean"],
                        "random reference": m[stat]["random"],
                        "limit from proportions": m[stat]["random_limit"],
                        "note": m["note"],
                    }
                )
show_table(pd.DataFrame(rows))
for stat, text in definitions.items():
    print(f"{stat}: {text}")
print(CAVEAT)

entropy: How evenly the batch labels are spread among each observation's 30 nearest neighbours, divided by log(min(30, number of labels)): 0 when all neighbours share one label, 1 when the labels are as evenly spread as the neighbours allow.
other_share: The share of each observation's 30 nearest neighbours that carry another batch.
Well mixed is not the same as biologically right: a correction can also mix cells that differ for a real reason, such as tissue or disease.

fallback_genes = {}
for entries in (marker_set.get("fallback") or {}).values():
    for comp, fb in (entries or {}).items():
        fallback_genes.setdefault(comp, set()).update(str(g).upper() for g in (fb or {}).get("genes") or [])
compartment_genes = {
    comp: {str(g).upper() for g in genes} | fallback_genes.get(comp, set())
    for comp, genes in marker_set["compartments"].items()
}
readings = {}
for name, v in views.items():
    key = "nn_harmony" if "nn_harmony_distances" in v.obsp else "nn_pca"
    leiden = {"resolution": 0.5, "random_state": SEED, "neighbors_key": key, "key_added": "leiden"}
    if use_gpu:  # rapids-singlecell's n_iterations counts aggregation levels, so its default is kept
        rsc.tl.leiden(v, **leiden)
    else:
        sc.tl.leiden(v, flavor="igraph", n_iterations=2, **leiden)
    expr = v.obsm["marker_expr"]
    m = ad.AnnData(
        X=expr.to_numpy(dtype=np.float32),
        obs=v.obs[["leiden"]].copy(),
        var=pd.DataFrame(index=expr.columns.astype(str)),
    )
    display(Markdown(f"View `{name}`: {m.obs['leiden'].nunique()} clusters on `{key}`, {m.n_vars} marker genes."))
    if m.n_vars == 0 or m.obs["leiden"].nunique() < 2:
        display(Markdown("This view has fewer than two clusters or no marker gene, so there is no dotplot."))
        continue
    by_compartment = {
        comp: [g for g in expr.columns if g.upper() in genes] for comp, genes in compartment_genes.items()
    }
    sc.pl.dotplot(m, {k: g for k, g in by_compartment.items() if g}, groupby="leiden", standard_scale="var")
    scores = (
        pd.DataFrame(
            {
                comp: expr[[g for g in expr.columns if g.upper() in genes]].mean(axis=1)
                for comp, genes in compartment_genes.items()
                if any(g.upper() in genes for g in expr.columns)
            }
        )
        .groupby(m.obs["leiden"].to_numpy())
        .mean()
    )
    ranked = scores.apply(lambda r: r.sort_values(ascending=False), axis=1)
    table = pd.DataFrame(
        {
            "observations": m.obs["leiden"].value_counts().reindex(scores.index),
            "top compartment": scores.idxmax(axis=1),
            "top mean": scores.max(axis=1),
            "runner-up": [r.dropna().index[1] if r.notna().sum() > 1 else "" for _, r in ranked.iterrows()],
            "runner-up mean": [r.dropna().iloc[1] if r.notna().sum() > 1 else np.nan for _, r in ranked.iterrows()],
        }
    ).rename_axis("cluster")
    readings[name] = table
    show_table(table)

for name, table in readings.items():
    lines = [f"View `{name}`:"]
    for cluster, r in table.iterrows():
        text = (
            f"- Cluster {cluster} ({int(r['observations'])} observations) has its highest mean marker expression "
            f"for {r['top compartment']} ({r['top mean']:.2f})"
        )
        if r["runner-up"]:
            text += f", followed by {r['runner-up']} ({r['runner-up mean']:.2f})"
        lines.append(text + ".")
    display(Markdown("\n".join(lines)))

show_call(
    export_bundle,
    "Writes the manifest, umap.parquet, qc.parquet, cohort.parquet and embedding.h5ad for this cohort. A cohort whose embedding is over 1 GiB gets parts of whole datasets instead (embedding-1.h5ad, embedding-2.h5ad, ...), and anndata.concat over the parts gives the whole cohort.",
)
bundle = export_bundle(
    views,
    cohort,
    qc,
    spec,
    out_dir,
    release,
    runtime_s=time.time() - start,
    mixing=mixing,
    primers_ref=primers_ref,
)
for f in sorted(bundle.iterdir()):
    print(f"{f.relative_to(out_dir)}  {f.stat().st_size} bytes")
manifest = json.loads((bundle / "manifest.json").read_text())
for key in (
    "slug",
    "release",
    "primers_ref",
    "n_datasets",
    "n_papers",
    "technologies",
    "n_cells_before",
    "n_cells_after",
    "n_flagged",
    "path",
    "hardware",
    "runtime_s",
):
    print(f"{key}: {manifest[key]}")

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.

command = (
    f"papermill primers/notebooks/primer.ipynb primer.ipynb -p slug {slug} -p release {release} "
    f"-p primers_ref {primers_ref} -p use_gpu {use_gpu}" + (f" -p smoke {smoke}" if smoke else "")
)
versions = "\n".join(f"  - {k} {v}" for k, v in manifest["versions"].items())
display(
    Markdown(
        f"- Catalog release: `{release}`\n- primers reference: `{primers_ref}`\n- Versions:\n{versions}\n"
        f"- Hardware: {manifest['hardware']} ({platform.system()})\n- Path: {manifest['path']}\n\n"
        "To run this primer again:"
    )
)
show_command(command)

papermill primers/notebooks/primer.ipynb primer.ipynb -p slug human-kidney -p release 2026-11-01 -p primers_ref 4d7af340849c00077f9d21c5b54af9e689a4a070 -p use_gpu True

	uid	title	technology	family	paper	paper_source	licence	n_obs	status
0	M7zdYIvV1fmS9H9i0000	GSM7974887	Visium	spot	10.1158/2159-8290.CD-24-0187	doi	ncbi-geo-policy	2258	kept
1	42EQMGTLwzdPRPKw0000	GSM7974477	Visium	spot	10.1038/s41467-024-52525-w	doi	ncbi-geo-policy	388	kept
2	D4I13ftxuL8b4qjP0000	human-kidney-11-mm-capture-area-ffpe-2-standard	Visium	spot	10x Genomics demo datasets	10x_demo	CC-BY-4.0	5936	kept
3	Fwp5w6FPox2J8RtV0000	GSM9210736	Xenium	imaging_cell	10.1016/j.celrep.2026.117483	doi	ncbi-geo-policy	34980	kept
4	jGRcwO1RMUByhiGB0000	GSM7115963	Visium	spot	geo-gse228169	geo_series	ncbi-geo-policy	3555	kept
...	...	...	...	...	...	...	...	...	...
152	VuiDeCXkuqd9M6B80000	GSM9844158	Xenium	imaging_cell	n/a	none	ncbi-geo-policy	13860	dropped: no paper (no study_id, series or collection)
153	Oe8QcqIOUO91Mkct0000	GSM9844164	Xenium	imaging_cell	n/a	none	ncbi-geo-policy	98714	dropped: no paper (no study_id, series or collection)
154	F6vuWevo1Qa8i2ob0000	GSM9844157	Xenium	imaging_cell	n/a	none	ncbi-geo-policy	23409	dropped: no paper (no study_id, series or collection)
155	EKqdvxicsy7bBSr70000	GSM9844156	Xenium	imaging_cell	n/a	none	ncbi-geo-policy	121864	dropped: no paper (no study_id, series or collection)
156	UwbrdvEmOWDDgSC30000	GSM9844159	Xenium	imaging_cell	n/a	none	ncbi-geo-policy	16807	dropped: no paper (no study_id, series or collection)

	dataset_uid	technology	n_before	n_after	median_counts	median_genes	pct_mt_median	flagged	flag_reason	status	family
0	0Y6QewKoDCORRZiZ0000	Visium	461	458	6550.0	3362.0	7.878	False		kept	spot
1	2Px4hjJFRq22eikg0000	Visium	2802	2798	15591.0	6647.0	2.342	False		kept	spot
2	2YwSMyV1o6mZlqSR0000	Visium	370	370	11169.5	4476.5	7.254	False		kept	spot
3	36rv58XjdrtRaHrW0000	Visium	1949	1946	17882.0	5285.0	4.446	False		kept	spot
4	42EQMGTLwzdPRPKw0000	Visium	388	380	13409.0	5748.0	0.0	False		kept	spot
...	...	...	...	...	...	...	...	...	...	...	...
139	mhUVgmN7Np9ERjpD0000	Xenium	54143	51796	56.0	30.0	n/a	False		kept	imaging_cell
140	mv6fU7wRDHoMkjEc0000	Xenium	26648	25811	73.0	38.0	n/a	False		kept	imaging_cell
141	oH98OgEjk6P33h620000	Xenium	44621	43365	69.0	37.0	n/a	False		kept	imaging_cell
142	qe9O7bVorvQTCEg00000	Xenium	44598	43575	88.0	44.0	n/a	False		kept	imaging_cell
143	vllMYzmHKoQQHzZ50000	Xenium	32584	25923	26.0	16.0	n/a	True	median counts 26 < 30	kept	imaging_cell

	markers
compartment
proximal_tubule	LRP2, CUBN, SLC34A1
thick_ascending_limb	UMOD, SLC12A1
distal_tubule	SLC12A3, CALB1
collecting_duct_principal	AQP2, AQP3
collecting_duct_intercalated	ATP6V1B1, SLC4A1
podocyte	NPHS1, NPHS2, WT1
endothelial	PECAM1, EMCN, PLVAP
fibroblast	DCN, COL1A1, PDGFRB
immune	PTPRC, CD68, CD3E

	panel	compartment	genes	note
0	Xenium-5092-381adb2d	collecting_duct_principal	SCNN1G, HSD11B2	principal-cell genes on this panel; GATA3 left out (also in distal segments)
1	Xenium-377-9193d582	proximal_tubule	SLC22A8	OAT3, proximal-tubule transporter on this panel
2	Xenium-377-9193d582	distal_tubule	PVALB	distal convoluted tubule gene on this panel
3	Xenium-377-9193d582	podocyte	not measured	no podocyte gene on this panel

	count floor
technology
Visium	500.0
SpatialTranscriptomics	500.0
VisiumHD	200.0
Xenium	10.0
Merscope	10.0
MERFISH	10.0

Cohort primer¶

Human kidney in health and disease¶

Setup¶

Query and cohort¶

Quality control¶

Marker set¶

Spatial overview¶

Embedding¶

Biology¶

Draft reading, to be reviewed by the Lead¶

Export and reproduce¶

	technologies	min_counts	min_genes	max_pct_mt	flag: min median counts	flag: min share kept	reason
group
spot	Visium, SpatialTranscriptomics	500	250	30.0	1000	0.5	A Visium or ST spot holds about 1 to 10 cells, so a spot with under 500 UMIs or 250 genes is mostly background or tissue edge; solid tissues often reach 20 to 30% mitochondrial reads, so the cap is 30%.
binned_spot	VisiumHD	200	50	30.0	200	0.5	On the 5 Visium HD sections of oliveira-2024-colorectal, 16 um bins at 200 counts kept 74% of bins with the same marker signal as 8 um bins at Oliveira et al.'s 100 (Nat Genet 57:1512-1523), which kept 60%, and found T cells in all 5 sections against 3 (web tasks/2026-11-01-t094-hd-bin-size.md).
imaging_cell	Xenium, Merscope, MERFISH	10	3	n/a	30	0.5	Targeted panels give tens to hundreds of decoded transcripts per segmented cell, and a cell with under 10 transcripts or 3 genes cannot be given a type; the panels have no mitochondrial genes.

	value	meaning
MIN_SHARED_GENES	200	genes a view needs before a family is split per technology, and the joint view's gate 1
MAX_FLOOR_LOSS	0.5	largest share of a technology's observations that may fall below its floor on the shared genes (gate 3)
N_HVG	2000	highly variable genes kept when a view has more genes
N_PCS	30	principal components
N_NEIGHBORS	30	neighbours per observation
SEED	0	random seed

	view	batch	embedding	statistic	mean	random reference	limit from proportions
0	spot	section	before Harmony	entropy	0.128	0.897	1.277
1	spot	section	before Harmony	other_share	0.195	0.944	0.977
2	spot	section	after Harmony	entropy	0.57	0.897	1.277
3	spot	section	after Harmony	other_share	0.729	0.944	0.977
4	spot	paper	before Harmony	entropy	0.063	0.73	0.835
5	spot	paper	before Harmony	other_share	0.097	0.854	0.883
6	spot	paper	after Harmony	entropy	0.369	0.729	0.835
7	spot	paper	after Harmony	other_share	0.532	0.854	0.883
8	imaging_cell	section	before Harmony	entropy	0.515	0.823	0.931
9	imaging_cell	section	before Harmony	other_share	0.63	0.88	0.909
10	imaging_cell	section	after Harmony	entropy	0.551	0.823	0.931
11	imaging_cell	section	after Harmony	other_share	0.651	0.88	0.909
12	imaging_cell	paper	before Harmony	entropy	0.325	0.725	0.766
13	imaging_cell	paper	before Harmony	other_share	0.265	0.566	0.586
14	imaging_cell	paper	after Harmony	entropy	0.387	0.728	0.766
15	imaging_cell	paper	after Harmony	other_share	0.313	0.567	0.586

	observations	top compartment	top mean	runner-up	runner-up mean
cluster
0	15896	fibroblast	0.951	collecting_duct_principal	0.575
1	44259	fibroblast	1.314	collecting_duct_principal	0.129
10	56689	fibroblast	0.757	collecting_duct_principal	0.111
2	13330	fibroblast	1.001	collecting_duct_principal	0.127
3	26201	fibroblast	0.492	collecting_duct_principal	0.342
4	13600	podocyte	0.999	collecting_duct_principal	0.368
5	4511	proximal_tubule	0.672	collecting_duct_principal	0.243
6	22614	thick_ascending_limb	1.485	collecting_duct_principal	0.634
7	52754	proximal_tubule	1.011	collecting_duct_principal	0.473
8	19488	collecting_duct_principal	1.411	collecting_duct_principal	1.411
9	6970	distal_tubule	1.662	collecting_duct_principal	0.555

	observations	top compartment	top mean	runner-up	runner-up mean
cluster
0	122964	endothelial	0.111	endothelial	0.111
1	158136	proximal_tubule	0.498	endothelial	0.032
10	21461	thick_ascending_limb	0.155	endothelial	0.058
100	19	proximal_tubule	0.0	endothelial	0.0
101	26	proximal_tubule	0.0	endothelial	0.0
...	...	...	...	...	...
95	14	proximal_tubule	0.0	endothelial	0.0
96	747	proximal_tubule	0.0	endothelial	0.0
97	84	proximal_tubule	0.0	endothelial	0.0
98	9	proximal_tubule	0.0	endothelial	0.0
99	21	proximal_tubule	0.0	endothelial	0.0