Source code for cellarium.ml.models.hvg_seurat_v3

# Copyright Contributors to the Cellarium project.
# SPDX-License-Identifier: BSD-3-Clause

from __future__ import annotations

import warnings
from typing import Literal

import lightning.pytorch as pl
import numpy as np
import pandas as pd
import skmisc.loess
import torch
import torch.distributed as dist
from lightning.pytorch.strategies import DDPStrategy

from cellarium.ml.core.datamodule import CellariumAnnDataDataModule
from cellarium.ml.models.model import CellariumModel
from cellarium.ml.utilities.distributed import get_rank_and_num_replicas
from cellarium.ml.utilities.testing import (
    assert_arrays_equal,
    assert_columns_and_array_lengths_equal,
)


def _fit_loess_with_jitter(
    x: np.ndarray,
    y: np.ndarray,
    span: float,
    max_jitter: float = 1e-6,
    initial_jitter: float = 1e-18,
    seed: int = 0,
) -> np.ndarray:
    rng = np.random.default_rng(seed)
    jitter = 0.0

    while jitter <= max_jitter:
        x_fit = x if jitter == 0 else x + rng.uniform(-jitter, jitter, size=x.shape[0])

        try:
            model = skmisc.loess.loess(x_fit, y, span=span, degree=2)
            model.fit()
            return model.outputs.fitted_values
        except ValueError:
            jitter = initial_jitter if jitter == 0 else jitter * 10

    raise ValueError(f"LOESS fit failed after retrying with jitter up to {max_jitter}.")


[docs] class HVGSeuratV3(CellariumModel): """ Compute highly variable genes using the Seurat v3 method in two Lightning epochs. **Epoch 0** — streams data to accumulate per-batch mean and variance. The sums are of the counts minus a per-batch, per-gene shift (the mean of the first cells seen from that batch), so that the variance is not computed as a difference of two large, nearly equal numbers. Between epochs — fits a LOESS model of ``log10(var) ~ log10(mean)`` per batch to estimate a regularized standard deviation and per-cell clip value. **Epoch 1** — streams data again, clips counts per cell at the batch-level ``clip_val``, and accumulates the sum of squared deviations of the clipped counts from the batch mean. After epoch 1 (``on_train_epoch_end``) — computes normalized variance per batch, ranks genes, and writes ``self.hvg_dfs`` (and optionally a CSV file at ``output_path``). Both epochs use only sums of positive terms or of shifted values, so the accumulators can be float32 (see ``accumulator_dtype``) with little loss of accuracy. The ``flavor`` argument controls how genes are ranked across batches when ``n_batch > 1``, matching the two multi-batch modes in Scanpy: * ``"seurat_v3"`` *(default)* — sorts by ``highly_variable_rank`` ascending first, then by ``highly_variable_nbatches`` descending as tiebreaker. Genes with the lowest median rank across batches are preferred. This matches Scanpy's ``flavor='seurat_v3'``. * ``"seurat_v3_paper"`` — sorts by ``highly_variable_nbatches`` descending first, then by ``highly_variable_rank`` ascending as tiebreaker. Genes that are highly variable in more batches are preferred, regardless of their exact rank. This matches Scanpy's ``flavor='seurat_v3_paper'``. Usage:: model = HVGSeuratV3(var_names_g=gene_names, n_top_genes=2000, n_batch=4, flavor="seurat_v3_paper", span=0.3) trainer = pl.Trainer(max_epochs=2) trainer.fit(module, datamodule) df = model.hvg_df # pandas DataFrame, Scanpy-compatible columns Args: var_names_g: Array of gene names, length ``n_genes``. n_top_genes: Number of highly variable genes to select. Can be a list of ints to produce multiple gene sets in a single training run. n_batch: Number of batches (use 1 when no batch information is given). flavor: Multi-batch gene ranking strategy. ``"seurat_v3_paper"`` (default) prioritises batch consistency; ``"seurat_v3"`` prioritises median rank. use_batch_key: Whether to expect a ``batch_index_n`` batch key in the dataloader output. If False, the model will ignore batch keys and treat all data as a single batch. span: LOESS span (fraction of data used per local fit). Default 0.3. output_path: If given, the result DataFrame is written to this CSV filepath after training. (Ends with ``.csv``). If ``None``, no file is written, but you really want to write this output, as it would require manually calling _compute_hvg_df() later. batch_n_cell_minimum: Minimum number of cells required for a batch to be considered valid. n_batch_minimum: Minimum number of batches in which the gene is in n_top_genes highly variable for a gene to make the final list. accumulator_dtype: The dtype of the accumulators, ``"float64"`` (default) or ``"float32"``. Use ``"float32"`` on devices without float64 support (MPS). """ _VALID_FLAVORS = frozenset({"seurat_v3", "seurat_v3_paper"}) def __init__( self, var_names_g: np.ndarray, n_top_genes: int | list[int], n_batch: int = 1, flavor: Literal["seurat_v3", "seurat_v3_paper"] = "seurat_v3", use_batch_key: bool = False, span: float = 0.3, output_path: str | None = "hvg_seurat_v3_output.csv", batch_n_cell_minimum: int = 2, n_batch_minimum: int = 1, accumulator_dtype: Literal["float32", "float64"] = "float64", ) -> None: super().__init__() if flavor not in self._VALID_FLAVORS: raise ValueError(f"flavor must be one of {sorted(self._VALID_FLAVORS)}, got {flavor!r}") if accumulator_dtype not in ("float32", "float64"): raise ValueError(f"accumulator_dtype must be 'float32' or 'float64', got {accumulator_dtype!r}") self.accumulator_dtype = accumulator_dtype dtype = getattr(torch, accumulator_dtype) self.var_names_g = var_names_g n_vars = len(var_names_g) self.n_vars = n_vars if isinstance(n_top_genes, int): self.n_top_genes_list = [n_top_genes] else: self.n_top_genes_list = sorted(set(n_top_genes)) self.n_batch = n_batch self.flavor = flavor self.use_batch_key = use_batch_key if use_batch_key and n_batch < 2: raise ValueError( "n_batch must be at least 2 when use_batch_key is True. This error may also be triggered " "if your dataloader is not providing the expected `batch_index_n` key: check your dataloader " "batch_keys and ensure `batch_index_n` is included when use_batch_key=True --" "data:" " ..." " batch_keys:" " ..." " batch_index_n:" " attr: obs" " key: my_categorical_batch_column" " convert_fn: cellarium.ml.utilities.data.categories_to_codes" ) self.span = span if (output_path is not None) and (not output_path.endswith(".csv")): raise ValueError("output_path must end with .csv") self.output_path = output_path self.batch_n_cell_minimum = batch_n_cell_minimum if n_batch_minimum > n_batch: raise ValueError(f"n_batch_minimum ({n_batch_minimum}) cannot exceed n_batch ({n_batch}).") self.n_batch_minimum = n_batch_minimum self._current_epoch: int = 0 # Epoch-0 buffers: shape (n_batch, n_vars). The sums are of ``x - x_shift_bg``, not of ``x``. # The cell count is int64 so that it stays exact past 2**24 cells (a float32 count stops incrementing there). self.x_shift_bg: torch.Tensor self.x_shift_set_b: torch.Tensor self.x_sums_bg: torch.Tensor self.x_squared_sums_bg: torch.Tensor self.x_size_b: torch.Tensor self.register_buffer("x_shift_bg", torch.zeros(n_batch, n_vars, dtype=dtype)) self.register_buffer("x_shift_set_b", torch.zeros(n_batch, dtype=torch.bool)) self.register_buffer("x_sums_bg", torch.zeros(n_batch, n_vars, dtype=dtype)) self.register_buffer("x_squared_sums_bg", torch.zeros(n_batch, n_vars, dtype=dtype)) self.register_buffer("x_size_b", torch.zeros(n_batch, dtype=torch.int64)) # whether every batch has a shift (a cache, to skip the work of finding shifts once they all are set) self._shifts_complete = False # Set between epochs by on_train_epoch_end after epoch 0: shape (n_batch, n_vars) self.mean_bg: torch.Tensor self.clip_val_bg: torch.Tensor self.reg_std_bg: torch.Tensor self.register_buffer("mean_bg", torch.zeros(n_batch, n_vars, dtype=dtype)) self.register_buffer("clip_val_bg", torch.zeros(n_batch, n_vars, dtype=dtype)) self.register_buffer("reg_std_bg", torch.zeros(n_batch, n_vars, dtype=dtype)) # Epoch-1 buffer: shape (n_batch, n_vars), the sum over cells of (clipped count - batch mean) ** 2 self.clipped_sq_dev_sum_bg: torch.Tensor self.register_buffer("clipped_sq_dev_sum_bg", torch.zeros(n_batch, n_vars, dtype=dtype)) # Dummy parameter so Lightning treats this as a trainable module self._dummy_param = torch.nn.Parameter(torch.empty(())) self._dummy_param.data.zero_() self.hvg_dfs: dict[int, pd.DataFrame] | None = None def reset_parameters(self) -> None: self.x_shift_bg.zero_() self.x_shift_set_b.zero_() self.x_sums_bg.zero_() self.x_squared_sums_bg.zero_() self.x_size_b.zero_() self.mean_bg.zero_() self.clip_val_bg.zero_() self.reg_std_bg.zero_() self.clipped_sq_dev_sum_bg.zero_() self._dummy_param.data.zero_() self._shifts_complete = False
[docs] def on_train_epoch_start(self, trainer: pl.Trainer) -> None: """Cache the current epoch number so forward() can branch on it.""" self._current_epoch = trainer.current_epoch
def forward( self, x_ng: torch.Tensor, var_names_g: np.ndarray, batch_index_n: torch.Tensor | None = None, ) -> dict[str, torch.Tensor | None]: assert_columns_and_array_lengths_equal("x_ng", x_ng, "var_names_g", var_names_g) assert_arrays_equal("var_names_g", var_names_g, "var_names_g", self.var_names_g) # help users avoid unintentional errors in configuring the use of batch_key if self.use_batch_key and (batch_index_n is None): raise ValueError( "batch_index_n is required when use_batch_key is True: add `batch_index_n` to your " "dataloader batch_keys." ) if (not self.use_batch_key) and (batch_index_n is not None): raise ValueError( "batch_index_n was given but use_batch_key is False: either set use_batch_key=True or remove " "batch_index_n from your dataloader batch_keys." ) if batch_index_n is None: batch_index_n = torch.zeros(x_ng.shape[0], dtype=torch.long, device=x_ng.device) else: batch_index_n = batch_index_n.long() # needed for scatter_add_ if self._current_epoch == 0: self._accumulate_epoch0(x_ng, batch_index_n) elif self._current_epoch == 1: self._accumulate_epoch1(x_ng, batch_index_n) else: raise RuntimeError(f"HVGSeuratV3 expects max_epochs=2, but got epoch {self._current_epoch}.") return {} def _set_new_shifts(self, x_ng: torch.Tensor, batch_idx_n: torch.Tensor) -> None: """ Set the shift of every batch that has cells in ``x_ng`` and no shift yet to the mean of those cells (over all processes, so that every process has the same shifts). The shift only has to be near the mean of the batch: any constant gives the same answer, but a near one avoids the loss of precision. """ if self._shifts_complete: return idx_exp_ng = batch_idx_n.unsqueeze(1).expand_as(x_ng) sums_bg = torch.zeros(self.n_batch, self.n_vars, dtype=x_ng.dtype, device=x_ng.device) sums_bg.scatter_add_(0, idx_exp_ng, x_ng) counts_b = torch.bincount(batch_idx_n, minlength=self.n_batch) _, world_size = get_rank_and_num_replicas() if world_size > 1: dist.all_reduce(sums_bg) dist.all_reduce(counts_b) new_b = (counts_b > 0) & ~self.x_shift_set_b mean_bg = sums_bg / counts_b.clamp(min=1).unsqueeze(1) self.x_shift_bg = torch.where(new_b.unsqueeze(1), mean_bg, self.x_shift_bg) self.x_shift_set_b = self.x_shift_set_b | new_b self._shifts_complete = bool(self.x_shift_set_b.all()) def _accumulate_epoch0(self, x_ng: torch.Tensor, batch_idx_n: torch.Tensor) -> None: x_ng = x_ng.to(self.x_sums_bg.dtype) self._set_new_shifts(x_ng, batch_idx_n) dev_ng = x_ng - self.x_shift_bg[batch_idx_n] idx_exp_ng = batch_idx_n.unsqueeze(1).expand_as(dev_ng) sums_contrib_bg = torch.zeros_like(self.x_sums_bg) sq_sums_contrib_bg = torch.zeros_like(self.x_squared_sums_bg) sums_contrib_bg.scatter_add_(0, idx_exp_ng, dev_ng) sq_sums_contrib_bg.scatter_add_(0, idx_exp_ng, dev_ng**2) self.x_sums_bg = self.x_sums_bg + sums_contrib_bg self.x_squared_sums_bg = self.x_squared_sums_bg + sq_sums_contrib_bg self.x_size_b = self.x_size_b + torch.bincount(batch_idx_n, minlength=self.n_batch) def _accumulate_epoch1(self, x_ng: torch.Tensor, batch_idx_n: torch.Tensor) -> None: x_ng = x_ng.to(self.clipped_sq_dev_sum_bg.dtype) # Per-cell clip value and batch mean: shape (n_cells, n_vars) x_clipped_ng = torch.minimum(x_ng, self.clip_val_bg[batch_idx_n]) dev_ng = x_clipped_ng - self.mean_bg[batch_idx_n] idx_exp_ng = batch_idx_n.unsqueeze(1).expand_as(dev_ng) sq_dev_contrib_bg = torch.zeros_like(self.clipped_sq_dev_sum_bg) sq_dev_contrib_bg.scatter_add_(0, idx_exp_ng, dev_ng**2) self.clipped_sq_dev_sum_bg = self.clipped_sq_dev_sum_bg + sq_dev_contrib_bg
[docs] def on_train_start(self, trainer: pl.Trainer) -> None: """Validation: if in a distributed setting, use DDP with broadcast_buffers=False.""" if trainer.world_size > 1: if not isinstance(trainer.strategy, DDPStrategy): raise ValueError("HVGSeuratV3 requires the DDP strategy.") if trainer.strategy._ddp_kwargs.get("broadcast_buffers") is not False: raise ValueError("HVGSeuratV3 requires broadcast_buffers=False.")
[docs] def on_train_epoch_end(self, trainer: pl.Trainer) -> None: """Implement the two-step Seurat v3 HVG method using hooks at the end of first and second epochs. After epoch 0, reduce buffers and fit LOESS to set clip_val_bg and reg_std_bg. After epoch 1, reduce buffers and compute hvg_df. """ if trainer.current_epoch == 0: self._finish_epoch0(trainer) elif trainer.current_epoch == 1: self._finish_epoch1(trainer)
def _finish_epoch0(self, trainer: pl.Trainer) -> None: # 1. Reduce epoch-0 buffers to rank 0 if trainer.world_size > 1: dist.reduce(self.x_sums_bg, dst=0, op=dist.ReduceOp.SUM) dist.reduce(self.x_squared_sums_bg, dst=0, op=dist.ReduceOp.SUM) dist.reduce(self.x_size_b, dst=0, op=dist.ReduceOp.SUM) # 2. Rank 0: compute mean/var per batch; fit LOESS; set clip_val_bg and reg_std_bg if trainer.global_rank == 0: self._compute_clip_val() # 3. Broadcast mean_bg, clip_val_bg and reg_std_bg to all ranks so epoch-1 can use them if trainer.world_size > 1: dist.broadcast(self.mean_bg, src=0) dist.broadcast(self.clip_val_bg, src=0) dist.broadcast(self.reg_std_bg, src=0) def _compute_clip_val(self) -> None: # Default to +inf so any batch skipped below (N < batch_n_cell_minimum) acts as a no-op clip. self.clip_val_bg.fill_(float("inf")) n_vars = self.n_vars for b in range(self.n_batch): N = self.x_size_b[b].item() if N < self.batch_n_cell_minimum: continue # the statistics are computed on the host in float64, from sums of the shifted counts shift_g = self.x_shift_bg[b].cpu().numpy().astype(np.float64) sums = self.x_sums_bg[b].cpu().numpy().astype(np.float64) sq_sums = self.x_squared_sums_bg[b].cpu().numpy().astype(np.float64) mean_g = shift_g + sums / N # Unbiased (sample) variance with Bessel's correction, matching Scanpy's correction=1 var_g = (sq_sums - sums**2 / N) / (N - 1) not_const = var_g > 0 estimated_var = np.zeros(n_vars, dtype=np.float64) if not_const.any(): x = np.log10(mean_g[not_const]) y = np.log10(var_g[not_const]) estimated_var[not_const] = _fit_loess_with_jitter(x, y, span=self.span) reg_std = np.sqrt(10**estimated_var) # shape (n_vars,) clip_val = reg_std * np.sqrt(N) + mean_g dtype, device = self.reg_std_bg.dtype, self.reg_std_bg.device self.mean_bg[b] = torch.as_tensor(mean_g, dtype=dtype, device=device) self.reg_std_bg[b] = torch.as_tensor(reg_std, dtype=dtype, device=device) self.clip_val_bg[b] = torch.as_tensor(clip_val, dtype=dtype, device=device) def _finish_epoch1(self, trainer: pl.Trainer) -> None: trainer.should_stop = True # 1. Reduce epoch-1 buffers to rank 0 if trainer.world_size > 1: dist.reduce(self.clipped_sq_dev_sum_bg, dst=0, op=dist.ReduceOp.SUM) # 2. Rank 0: compute norm_var, rank, select top genes, build DataFrame and optionally save to CSV if trainer.global_rank == 0: # Retrieve adata.var annotation columns to enrich the output DataFrame. var_df: pd.DataFrame | None = None datamodule = getattr(trainer, "datamodule", None) if isinstance(datamodule, CellariumAnnDataDataModule): var_df = datamodule.dadc.var else: warnings.warn( "HVGSeuratV3: trainer.datamodule is not a CellariumAnnDataDataModule; " "adata.var annotations will not be added to the output DataFrame.", UserWarning, stacklevel=2, ) self.hvg_dfs = {} for n in self.n_top_genes_list: df = self._compute_hvg_df(n_top_genes=n, var_df=var_df) self.hvg_dfs[n] = df if self.output_path is not None: path = self.output_path.replace(".csv", f"__top{n}.csv") self._save(df=df, output_path=path) def _compute_hvg_df(self, n_top_genes: int, var_df: pd.DataFrame | None = None) -> pd.DataFrame: n_vars = self.n_vars norm_gene_var_bg = np.zeros((self.n_batch, n_vars), dtype=np.float64) for b in range(self.n_batch): N = self.x_size_b[b].item() if N < self.batch_n_cell_minimum: continue reg_std_g = self.reg_std_bg[b].cpu().numpy().astype(np.float64) sq_dev_sum_g = self.clipped_sq_dev_sum_bg[b].cpu().numpy().astype(np.float64) denom_g = (N - 1) * reg_std_g**2 with np.errstate(divide="ignore", invalid="ignore"): norm_gene_var_g = sq_dev_sum_g / denom_g norm_gene_var_g[np.isnan(norm_gene_var_g)] = 0.0 norm_gene_var_bg[b] = norm_gene_var_g # Only rank over batches with sufficient data (N >= batch_n_cell_minimum); invalid batches # have norm_gene_vars == 0 and would otherwise pollute num_batches_high_var # and median_ranked with arbitrary rankings of equal values. valid_b = np.array([self.x_size_b[b].item() >= self.batch_n_cell_minimum for b in range(self.n_batch)]) norm_gene_vars_valid_vg = norm_gene_var_bg[valid_b] # Rank genes within each valid batch v # (genes with identical counts in a batch tie exactly; a stable sort breaks the tie by gene order, instead of # by whatever the unstable sort does with the rounding noise of the other genes) hvg_rank_vg = np.argsort(np.argsort(-norm_gene_vars_valid_vg, axis=1, kind="stable"), axis=1).astype(np.float32) num_batches_high_var_v = (hvg_rank_vg < n_top_genes).sum(axis=0).astype(int) hvg_rank_vg[hvg_rank_vg >= n_top_genes] = np.nan masked_hvg_rank_vg = np.ma.masked_invalid(hvg_rank_vg) median_hvg_rank_g = np.ma.median(masked_hvg_rank_vg, axis=0).filled(np.nan) variances_norm_g = norm_gene_vars_valid_vg.mean(axis=0) df = pd.DataFrame( index=pd.Index(self.var_names_g, name="gene"), data={ "highly_variable_nbatches": num_batches_high_var_v, "highly_variable_rank": median_hvg_rank_g, "variances_norm": variances_norm_g, }, ) # Use integer-position sort so duplicate gene names don't cause # df.loc to mark more than n_top_genes rows as highly variable. rank_vals_g = df["highly_variable_rank"].fillna(np.inf).values nbatches_vals_g = df["highly_variable_nbatches"].values # np.lexsort: LAST key = primary sort key. if self.flavor == "seurat_v3": # Primary: rank ascending; tiebreaker: nbatches descending. sort_positions_g = np.lexsort((-nbatches_vals_g, rank_vals_g)) else: # seurat_v3_paper # Primary: nbatches descending; tiebreaker: rank ascending. sort_positions_g = np.lexsort((rank_vals_g, -nbatches_vals_g)) eligible_g = df["highly_variable_nbatches"].values >= self.n_batch_minimum eligible_in_sort_order_g = sort_positions_g[eligible_g[sort_positions_g]] hvg_flags = np.zeros(len(df), dtype=bool) hvg_flags[eligible_in_sort_order_g[:n_top_genes]] = True df["highly_variable"] = hvg_flags if self.n_batch == 1: df = df.drop(columns=["highly_variable_nbatches"]) # Append any extra columns from adata.var (e.g. gene_id, gene_name). # df.index values are var_names_g; var_df.index is adata.var_names. # pandas .join() aligns on label values, so index *name* differences # are harmless, and genes absent from var_df receive NaN. if var_df is not None: extra_cols = var_df.columns.difference(df.columns) if len(extra_cols) > 0: df = df.join(var_df[extra_cols], how="left") df = df.iloc[sort_positions_g] # reorder rows by rank return df def _save(self, df: pd.DataFrame, output_path: str) -> None: df.to_csv(output_path) hvg_df = df[df["highly_variable"]] hvg_df.to_csv(output_path.replace(".csv", "__hvg_only.csv"), index=True, header=True)